diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-80000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-80000.pth new file mode 100644 index 0000000000000000000000000000000000000000..89f1e3545ebdb1ee95f7c046a7649842c52738b6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-80000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bd103bd1ffb19c2adabbea14d210620f3de3f70511d40322aecd6451f6f9ee0 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..1845c1f176ac471f1027360d522f95573f2ff611 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 4.015713691711426, "Classification": 0.03849282115697861, "Regression_pt": 0.1724710911512375, "Regression_eta": 0.008294960483908653, "Regression_sin_phi": 0.012356956489384174, "Regression_cos_phi": 0.009977273643016815, "Regression_energy": 0.16784390807151794, "Total": 8.970200538635254, "Total_unweighted": 4.4251507027074695}, "train_group": {}, "valid": {"Classification_binary": 3.9935059547424316, "Classification": 0.03709951415657997, "Regression_pt": 0.21856904029846191, "Regression_eta": 0.01491805911064148, "Regression_sin_phi": 0.010844285599887371, "Regression_cos_phi": 0.01119080837816, "Regression_energy": 0.21647045016288757, "Total": 10.150409698486328, "Total_unweighted": 4.50259811244905, "metrics/particle/class_1/efficiency": 0.8915127515792847, "metrics/particle/class_1/pid_accuracy": 0.98338383436203, "metrics/particle/class_2/efficiency": 0.17603211104869843, "metrics/particle/class_2/pid_accuracy": 0.3094462454319, "metrics/particle/class_3/efficiency": 0.6996718645095825, "metrics/particle/class_3/pid_accuracy": 0.9764137864112854, "metrics/particle/class_4/efficiency": 0.85467129945755, "metrics/particle/class_4/pid_accuracy": 0.028340080752968788, "metrics/particle/class_5/efficiency": 0.9427083134651184, "metrics/particle/class_5/pid_accuracy": 0.0883977934718132, "metrics/particle/count/bias_mean": -0.23199999332427979, "metrics/particle/count/mae": 5.136000156402588, "metrics/particle/count/prediction_mean": 96.0199966430664, "metrics/particle/count/target_mean": 96.25199890136719, "metrics/particle/event/energy_relative_abs_error": 0.08249011635780334, "metrics/particle/event/energy_response_mean": 0.9582615494728088, "metrics/particle/event/met_abs_error": 12.815853118896484, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08131930232048035, "metrics/particle/event/scalar_pt_response_mean": 0.9598265886306763, "metrics/particle/event/vector_pt_closure": 0.08490834385156631, "metrics/particle/matched/delta_eta_abs_mean": 0.021934427320957184, "metrics/particle/matched/delta_phi_abs_mean": 0.01965649053454399, "metrics/particle/matched/delta_r_mean": 0.03280063718557358, "metrics/particle/matched/energy_relative_abs_mean": 0.09646397083997726, "metrics/particle/matched/pid_accuracy": 0.9474088549613953, "metrics/particle/matched/pt_relative_abs_mean": 0.09171808511018753, "metrics/particle/matching/duplicate_fraction": 0.04269943758845329, "metrics/particle/matching/efficiency": 0.7570128440856934, "metrics/particle/matching/f1": 0.7579262852668762, "metrics/particle/matching/purity": 0.7588419318199158, "metrics/particle/matching/target_coverage_dr0p05": 0.624610424041748, "metrics/particle/matching/target_coverage_dr0p10": 0.789677083492279, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6719028949737549, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.81727135181427}, "step_train_time": 0.7900326251983643, "step_valid_time": 1.851197028066963, "step_total_time": 740.6127593517303, "elapsed_training_loop_seconds": 740.6127593517303, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9760879499758485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20315851103941407, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1490, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1296, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1226, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8715534633490248, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8697986577181208, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8706751763520323, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13020134228187918, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8244788164088769, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8228187919463087, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8236479677527713, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8715534633490248, "step_test_time": 3.1267466300632805, "step_plot_time": 2.3640741700073704, "step_validation_cycle_time": 7.8253866960294545} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..f577a7b1d783fc71b64fb7c6b5196d9e97a65024 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 3.85414981842041, "Classification": 0.039956022053956985, "Regression_pt": 0.21875329315662384, "Regression_eta": 0.013725532218813896, "Regression_sin_phi": 0.00931147113442421, "Regression_cos_phi": 0.008958750404417515, "Regression_energy": 0.23187901079654694, "Total": 10.161786079406738, "Total_unweighted": 4.3767338981851935}, "train_group": {}, "valid": {"Classification_binary": 3.8532674312591553, "Classification": 0.03542546555399895, "Regression_pt": 0.20570704340934753, "Regression_eta": 0.013729248195886612, "Regression_sin_phi": 0.009950397536158562, "Regression_cos_phi": 0.010331022553145885, "Regression_energy": 0.2029266059398651, "Total": 9.637345314025879, "Total_unweighted": 4.331337214447558, "metrics/particle/class_1/efficiency": 0.8859358429908752, "metrics/particle/class_1/pid_accuracy": 0.9806236028671265, "metrics/particle/class_2/efficiency": 0.21731650829315186, "metrics/particle/class_2/pid_accuracy": 0.4142480194568634, "metrics/particle/class_3/efficiency": 0.7200347185134888, "metrics/particle/class_3/pid_accuracy": 0.978823184967041, "metrics/particle/class_4/efficiency": 0.8719723224639893, "metrics/particle/class_4/pid_accuracy": 0.0833333358168602, "metrics/particle/class_5/efficiency": 0.9375, "metrics/particle/class_5/pid_accuracy": 0.23888888955116272, "metrics/particle/count/bias_mean": -0.052000001072883606, "metrics/particle/count/mae": 4.699999809265137, "metrics/particle/count/prediction_mean": 96.19999694824219, "metrics/particle/count/target_mean": 96.25199890136719, "metrics/particle/event/energy_relative_abs_error": 0.07599479705095291, "metrics/particle/event/energy_response_mean": 0.9820490479469299, "metrics/particle/event/met_abs_error": 11.6581449508667, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07450247555971146, "metrics/particle/event/scalar_pt_response_mean": 0.9842734336853027, "metrics/particle/event/vector_pt_closure": 0.07765036821365356, "metrics/particle/matched/delta_eta_abs_mean": 0.019137511029839516, "metrics/particle/matched/delta_phi_abs_mean": 0.017932046204805374, "metrics/particle/matched/delta_r_mean": 0.02917354926466942, "metrics/particle/matched/energy_relative_abs_mean": 0.09705743938684464, "metrics/particle/matched/pid_accuracy": 0.9487499594688416, "metrics/particle/matched/pt_relative_abs_mean": 0.09895728528499603, "metrics/particle/matching/duplicate_fraction": 0.04049896076321602, "metrics/particle/matching/efficiency": 0.7662801742553711, "metrics/particle/matching/f1": 0.7664872407913208, "metrics/particle/matching/purity": 0.7666943669319153, "metrics/particle/matching/target_coverage_dr0p05": 0.6633005142211914, "metrics/particle/matching/target_coverage_dr0p10": 0.8033080101013184, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6701574921607971, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8215933442115784}, "step_train_time": 0.7584407329559326, "step_valid_time": 1.9819169419351965, "step_total_time": 1494.6416606903076, "elapsed_training_loop_seconds": 1494.6416606903076, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0025417175716829, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20605151054420978, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1494, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1322, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1267, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8890383322125084, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8848728246318608, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8869506876886951, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11512717536813923, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8520511096166778, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8480589022757697, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.850050318685005, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8890383322125084, "step_test_time": 3.2108992770081386, "step_plot_time": 2.262435681070201, "step_validation_cycle_time": 7.951159760938026} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..6d441dd4e5893b9d7e3867ea0c72d6bc29498700 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 3.4128012657165527, "Classification": 0.028861206024885178, "Regression_pt": 0.16473138332366943, "Regression_eta": 0.00788995809853077, "Regression_sin_phi": 0.00824070442467928, "Regression_cos_phi": 0.005993984639644623, "Regression_energy": 0.1638282686471939, "Total": 8.017716407775879, "Total_unweighted": 3.792346770875156}, "train_group": {}, "valid": {"Classification_binary": 3.730581045150757, "Classification": 0.03444462642073631, "Regression_pt": 0.20082174241542816, "Regression_eta": 0.012932317331433296, "Regression_sin_phi": 0.009554648771882057, "Regression_cos_phi": 0.009967920370399952, "Regression_energy": 0.19831660389900208, "Total": 9.3712739944458, "Total_unweighted": 4.196618904359639, "metrics/particle/class_1/efficiency": 0.8915998339653015, "metrics/particle/class_1/pid_accuracy": 0.9773260354995728, "metrics/particle/class_2/efficiency": 0.2626146674156189, "metrics/particle/class_2/pid_accuracy": 0.4519650638103485, "metrics/particle/class_3/efficiency": 0.7423277497291565, "metrics/particle/class_3/pid_accuracy": 0.9795891642570496, "metrics/particle/class_4/efficiency": 0.8615916967391968, "metrics/particle/class_4/pid_accuracy": 0.09236947447061539, "metrics/particle/class_5/efficiency": 0.9479166865348816, "metrics/particle/class_5/pid_accuracy": 0.26923078298568726, "metrics/particle/count/bias_mean": 1.9880000352859497, "metrics/particle/count/mae": 5.124000072479248, "metrics/particle/count/prediction_mean": 98.23999786376953, "metrics/particle/count/target_mean": 96.25199890136719, "metrics/particle/event/energy_relative_abs_error": 0.07641645520925522, "metrics/particle/event/energy_response_mean": 1.0525538921356201, "metrics/particle/event/met_abs_error": 10.3040771484375, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0786113515496254, "metrics/particle/event/scalar_pt_response_mean": 1.0563106536865234, "metrics/particle/event/vector_pt_closure": 0.06726865470409393, "metrics/particle/matched/delta_eta_abs_mean": 0.019353386014699936, "metrics/particle/matched/delta_phi_abs_mean": 0.017648063600063324, "metrics/particle/matched/delta_r_mean": 0.029219524934887886, "metrics/particle/matched/energy_relative_abs_mean": 0.10384313762187958, "metrics/particle/matched/pid_accuracy": 0.9468984007835388, "metrics/particle/matched/pt_relative_abs_mean": 0.104012131690979, "metrics/particle/matching/duplicate_fraction": 0.04405537620186806, "metrics/particle/matching/efficiency": 0.7818227410316467, "metrics/particle/matching/f1": 0.773831307888031, "metrics/particle/matching/purity": 0.7660016417503357, "metrics/particle/matching/target_coverage_dr0p05": 0.6711548566818237, "metrics/particle/matching/target_coverage_dr0p10": 0.814653217792511, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6742717027664185, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8385488390922546}, "step_train_time": 0.7786905765533447, "step_valid_time": 1.9741058710496873, "step_total_time": 2247.90217089653, "elapsed_training_loop_seconds": 2247.90217089653, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.042160206545072, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19340132982669933, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1334, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1288, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8971082716879624, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8917112299465241, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8944016091183372, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10828877005347594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8661735036987223, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8609625668449198, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8635601743211532, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8971082716879624, "step_test_time": 3.15345503797289, "step_plot_time": 2.269668795983307, "step_validation_cycle_time": 7.895503300009295} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..dfeca373294151e6ec47a4107c28159a56f83b2e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 3.3526248931884766, "Classification": 0.03167172893881798, "Regression_pt": 0.1584295928478241, "Regression_eta": 0.01037686225026846, "Regression_sin_phi": 0.013397022150456905, "Regression_cos_phi": 0.00832261424511671, "Regression_energy": 0.15761804580688477, "Total": 7.934598922729492, "Total_unweighted": 3.7324407594278455}, "train_group": {}, "valid": {"Classification_binary": 3.5372915267944336, "Classification": 0.03095990978181362, "Regression_pt": 0.18995508551597595, "Regression_eta": 0.012768946588039398, "Regression_sin_phi": 0.009282544255256653, "Regression_cos_phi": 0.009608320891857147, "Regression_energy": 0.18941877782344818, "Total": 8.88351058959961, "Total_unweighted": 3.9792851116508245, "metrics/particle/class_1/efficiency": 0.9007493853569031, "metrics/particle/class_1/pid_accuracy": 0.9783302545547485, "metrics/particle/class_2/efficiency": 0.28153669834136963, "metrics/particle/class_2/pid_accuracy": 0.5804480910301208, "metrics/particle/class_3/efficiency": 0.7419416904449463, "metrics/particle/class_3/pid_accuracy": 0.9798387289047241, "metrics/particle/class_4/efficiency": 0.8685120940208435, "metrics/particle/class_4/pid_accuracy": 0.19521912932395935, "metrics/particle/class_5/efficiency": 0.953125, "metrics/particle/class_5/pid_accuracy": 0.34972676634788513, "metrics/particle/count/bias_mean": 1.6319999694824219, "metrics/particle/count/mae": 4.455999851226807, "metrics/particle/count/prediction_mean": 97.88400268554688, "metrics/particle/count/target_mean": 96.25199890136719, "metrics/particle/event/energy_relative_abs_error": 0.05252637714147568, "metrics/particle/event/energy_response_mean": 1.0114774703979492, "metrics/particle/event/met_abs_error": 8.625664710998535, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05427943542599678, "metrics/particle/event/scalar_pt_response_mean": 1.0122069120407104, "metrics/particle/event/vector_pt_closure": 0.056626804172992706, "metrics/particle/matched/delta_eta_abs_mean": 0.019263634458184242, "metrics/particle/matched/delta_phi_abs_mean": 0.017914200201630592, "metrics/particle/matched/delta_r_mean": 0.02931937947869301, "metrics/particle/matched/energy_relative_abs_mean": 0.10223738104104996, "metrics/particle/matched/pid_accuracy": 0.9521899819374084, "metrics/particle/matched/pt_relative_abs_mean": 0.0932522714138031, "metrics/particle/matching/duplicate_fraction": 0.041641127318143845, "metrics/particle/matching/efficiency": 0.7875161170959473, "metrics/particle/matching/f1": 0.7808958888053894, "metrics/particle/matching/purity": 0.7743859887123108, "metrics/particle/matching/target_coverage_dr0p05": 0.6699497103691101, "metrics/particle/matching/target_coverage_dr0p10": 0.8190998435020447, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6955491900444031, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8466525077819824}, "step_train_time": 0.7785544395446777, "step_valid_time": 2.0094264149665833, "step_total_time": 3005.2798953056335, "elapsed_training_loop_seconds": 3005.2798953056335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9999089393164721, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15658918325822802, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1475, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1343, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1317, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9031607262945528, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9105084745762712, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9068197164078325, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08949152542372882, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8856758574310692, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8928813559322034, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8892640108035111, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9031607262945528, "step_test_time": 3.144862362067215, "step_plot_time": 2.2593824999639764, "step_validation_cycle_time": 7.923551031039096} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..59b916d246f30ac1906cb074fd67251b7edb6b65 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 3.421208620071411, "Classification": 0.02759786881506443, "Regression_pt": 0.16131553053855896, "Regression_eta": 0.004970055539160967, "Regression_sin_phi": 0.006630134303122759, "Regression_cos_phi": 0.005987836513668299, "Regression_energy": 0.16307257115840912, "Total": 7.915924072265625, "Total_unweighted": 3.7907826169393957}, "train_group": {}, "valid": {"Classification_binary": 3.4951415061950684, "Classification": 0.029393630102276802, "Regression_pt": 0.185575932264328, "Regression_eta": 0.012925829738378525, "Regression_sin_phi": 0.008846519514918327, "Regression_cos_phi": 0.009072529152035713, "Regression_energy": 0.18534675240516663, "Total": 8.709552764892578, "Total_unweighted": 3.9263026993721724, "metrics/particle/class_1/efficiency": 0.9005751013755798, "metrics/particle/class_1/pid_accuracy": 0.9751330614089966, "metrics/particle/class_2/efficiency": 0.2780963182449341, "metrics/particle/class_2/pid_accuracy": 0.5773195624351501, "metrics/particle/class_3/efficiency": 0.7369233965873718, "metrics/particle/class_3/pid_accuracy": 0.97826087474823, "metrics/particle/class_4/efficiency": 0.8615916967391968, "metrics/particle/class_4/pid_accuracy": 0.1807228922843933, "metrics/particle/class_5/efficiency": 0.9635416865348816, "metrics/particle/class_5/pid_accuracy": 0.4810810685157776, "metrics/particle/count/bias_mean": 1.496000051498413, "metrics/particle/count/mae": 4.567999839782715, "metrics/particle/count/prediction_mean": 97.74800109863281, "metrics/particle/count/target_mean": 96.25199890136719, "metrics/particle/event/energy_relative_abs_error": 0.051292069256305695, "metrics/particle/event/energy_response_mean": 1.0185492038726807, "metrics/particle/event/met_abs_error": 9.203628540039062, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05185657739639282, "metrics/particle/event/scalar_pt_response_mean": 1.0204607248306274, "metrics/particle/event/vector_pt_closure": 0.06111479178071022, "metrics/particle/matched/delta_eta_abs_mean": 0.018823673948645592, "metrics/particle/matched/delta_phi_abs_mean": 0.017793312668800354, "metrics/particle/matched/delta_r_mean": 0.02889835275709629, "metrics/particle/matched/energy_relative_abs_mean": 0.09974927455186844, "metrics/particle/matched/pid_accuracy": 0.9508734941482544, "metrics/particle/matched/pt_relative_abs_mean": 0.09472235292196274, "metrics/particle/matching/duplicate_fraction": 0.04386790469288826, "metrics/particle/matching/efficiency": 0.7850226759910583, "metrics/particle/matching/f1": 0.7789690494537354, "metrics/particle/matching/purity": 0.7730081677436829, "metrics/particle/matching/target_coverage_dr0p05": 0.6750197410583496, "metrics/particle/matching/target_coverage_dr0p10": 0.8213024139404297, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.689481794834137, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8418318629264832}, "step_train_time": 0.7675926685333252, "step_valid_time": 2.0194027920952067, "step_total_time": 3759.2077395915985, "elapsed_training_loop_seconds": 3759.2077395915985, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0086475019767045, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14757668473335928, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1349, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1325, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9071956960322798, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9108710330857529, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9090296495956873, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08912896691424713, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8910558170813719, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8946657663740716, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8928571428571429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9071956960322798, "step_test_time": 3.2380416449159384, "step_plot_time": 2.2700440130429342, "step_validation_cycle_time": 8.037020783987828} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_60000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_60000.json new file mode 100644 index 0000000000000000000000000000000000000000..25345caa6fc42f8f0bea7b766790b09a3e3087f0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_60000.json @@ -0,0 +1 @@ +{"epoch": 106.76156583629893, "train": {"Classification_binary": 3.1106514930725098, "Classification": 0.0271860770881176, "Regression_pt": 0.1168924868106842, "Regression_eta": 0.01179321389645338, "Regression_sin_phi": 0.009063358418643475, "Regression_cos_phi": 0.00981047935783863, "Regression_energy": 0.12327231466770172, "Total": 6.673815727233887, "Total_unweighted": 3.4086694233119488}, "train_group": {}, "valid": {"Classification_binary": 3.517404317855835, "Classification": 0.029017440974712372, "Regression_pt": 0.1943494826555252, "Regression_eta": 0.012896087020635605, "Regression_sin_phi": 0.008738631382584572, "Regression_cos_phi": 0.008890822529792786, "Regression_energy": 0.1944778710603714, "Total": 8.945260047912598, "Total_unweighted": 3.965774653479457, "metrics/particle/class_1/efficiency": 0.9105960130691528, "metrics/particle/class_1/pid_accuracy": 0.9781818389892578, "metrics/particle/class_2/efficiency": 0.26146790385246277, "metrics/particle/class_2/pid_accuracy": 0.5460526347160339, "metrics/particle/class_3/efficiency": 0.7292028665542603, "metrics/particle/class_3/pid_accuracy": 0.9769719243049622, "metrics/particle/class_4/efficiency": 0.875432550907135, "metrics/particle/class_4/pid_accuracy": 0.2134387344121933, "metrics/particle/class_5/efficiency": 0.953125, "metrics/particle/class_5/pid_accuracy": 0.4590163826942444, "metrics/particle/count/bias_mean": 0.164000004529953, "metrics/particle/count/mae": 4.699999809265137, "metrics/particle/count/prediction_mean": 96.41600036621094, "metrics/particle/count/target_mean": 96.25199890136719, "metrics/particle/event/energy_relative_abs_error": 0.05963905155658722, "metrics/particle/event/energy_response_mean": 0.9825883507728577, "metrics/particle/event/met_abs_error": 9.605710983276367, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05837829038500786, "metrics/particle/event/scalar_pt_response_mean": 0.9896187782287598, "metrics/particle/event/vector_pt_closure": 0.06317146122455597, "metrics/particle/matched/delta_eta_abs_mean": 0.019648971036076546, "metrics/particle/matched/delta_phi_abs_mean": 0.017693988978862762, "metrics/particle/matched/delta_r_mean": 0.029439376667141914, "metrics/particle/matched/energy_relative_abs_mean": 0.09492231905460358, "metrics/particle/matched/pid_accuracy": 0.9520055055618286, "metrics/particle/matched/pt_relative_abs_mean": 0.0910792201757431, "metrics/particle/matching/duplicate_fraction": 0.039080649614334106, "metrics/particle/matching/efficiency": 0.7853550910949707, "metrics/particle/matching/f1": 0.7846866250038147, "metrics/particle/matching/purity": 0.7840192317962646, "metrics/particle/matching/target_coverage_dr0p05": 0.6628433465957642, "metrics/particle/matching/target_coverage_dr0p10": 0.8155674934387207, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6981257796287537, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8408344984054565}, "step_train_time": 0.7721717357635498, "step_valid_time": 2.0387332050595433, "step_total_time": 4518.969594478607, "elapsed_training_loop_seconds": 4518.969594478607, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9936132448009287, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14660449708526035, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1475, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1334, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1301, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8971082716879624, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9044067796610169, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.900742741390952, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09559322033898306, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.874915938130464, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8820338983050847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8784604996623903, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8971082716879624, "step_test_time": 3.1144363019848242, "step_plot_time": 2.280186887946911, "step_validation_cycle_time": 7.903902680031024} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_70000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_70000.json new file mode 100644 index 0000000000000000000000000000000000000000..11c61a62bb5b62500ae6dd9eec78222ceedab66d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_70000.json @@ -0,0 +1 @@ +{"epoch": 124.55516014234875, "train": {"Classification_binary": 2.873628616333008, "Classification": 0.02559659257531166, "Regression_pt": 0.09358017891645432, "Regression_eta": 0.013041323982179165, "Regression_sin_phi": 0.014536046423017979, "Regression_cos_phi": 0.014460190199315548, "Regression_energy": 0.10534615814685822, "Total": 6.020669460296631, "Total_unweighted": 3.1401891065761447}, "train_group": {}, "valid": {"Classification_binary": 3.6400890350341797, "Classification": 0.029988231137394905, "Regression_pt": 0.21184934675693512, "Regression_eta": 0.012796427123248577, "Regression_sin_phi": 0.008688784204423428, "Regression_cos_phi": 0.009051826782524586, "Regression_energy": 0.21239231526851654, "Total": 9.516793251037598, "Total_unweighted": 4.124855966307223, "metrics/particle/class_1/efficiency": 0.9065876603126526, "metrics/particle/class_1/pid_accuracy": 0.9786620736122131, "metrics/particle/class_2/efficiency": 0.27694955468177795, "metrics/particle/class_2/pid_accuracy": 0.6211180090904236, "metrics/particle/class_3/efficiency": 0.7238950133323669, "metrics/particle/class_3/pid_accuracy": 0.9745367169380188, "metrics/particle/class_4/efficiency": 0.8650519251823425, "metrics/particle/class_4/pid_accuracy": 0.18000000715255737, "metrics/particle/class_5/efficiency": 0.9635416865348816, "metrics/particle/class_5/pid_accuracy": 0.4810810685157776, "metrics/particle/count/bias_mean": -0.2879999876022339, "metrics/particle/count/mae": 5.176000118255615, "metrics/particle/count/prediction_mean": 95.96399688720703, "metrics/particle/count/target_mean": 96.25199890136719, "metrics/particle/event/energy_relative_abs_error": 0.05896647647023201, "metrics/particle/event/energy_response_mean": 0.9905875325202942, "metrics/particle/event/met_abs_error": 9.164907455444336, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05424301698803902, "metrics/particle/event/scalar_pt_response_mean": 0.9954108595848083, "metrics/particle/event/vector_pt_closure": 0.06066562607884407, "metrics/particle/matched/delta_eta_abs_mean": 0.01872519962489605, "metrics/particle/matched/delta_phi_abs_mean": 0.01775435172021389, "metrics/particle/matched/delta_r_mean": 0.02873760648071766, "metrics/particle/matched/energy_relative_abs_mean": 0.0918944850564003, "metrics/particle/matched/pid_accuracy": 0.9523455500602722, "metrics/particle/matched/pt_relative_abs_mean": 0.08879776298999786, "metrics/particle/matching/duplicate_fraction": 0.0393064059317112, "metrics/particle/matching/efficiency": 0.7822383046150208, "metrics/particle/matching/f1": 0.7834103107452393, "metrics/particle/matching/purity": 0.7845858931541443, "metrics/particle/matching/target_coverage_dr0p05": 0.6610979437828064, "metrics/particle/matching/target_coverage_dr0p10": 0.812450647354126, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6977517604827881, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8395046591758728}, "step_train_time": 0.7769639492034912, "step_valid_time": 2.0364475160604343, "step_total_time": 5276.050372123718, "elapsed_training_loop_seconds": 5276.050372123718, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988843370443912, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15548436553221368, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1466, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1336, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1310, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.898453261600538, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9113233287858117, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9048425330172706, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08867667121418826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8809683927370545, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8935879945429741, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8872333220453776, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.898453261600538, "step_test_time": 3.1147123189875856, "step_plot_time": 2.2693937179865316, "step_validation_cycle_time": 7.891613306012005} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_80000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_80000.json new file mode 100644 index 0000000000000000000000000000000000000000..dd86481b655fab542ac610963116796542ad67f7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/history/step_80000.json @@ -0,0 +1 @@ +{"epoch": 142.34875444839858, "train": {"Classification_binary": 2.3644888401031494, "Classification": 0.024062860757112503, "Regression_pt": 0.06747335195541382, "Regression_eta": 0.005027283914387226, "Regression_sin_phi": 0.009431242011487484, "Regression_cos_phi": 0.00709893461316824, "Regression_energy": 0.06828737258911133, "Total": 4.529984474182129, "Total_unweighted": 2.54586988594383}, "train_group": {}, "valid": {"Classification_binary": 4.056398868560791, "Classification": 0.0314062237739563, "Regression_pt": 0.22484923899173737, "Regression_eta": 0.013039530254900455, "Regression_sin_phi": 0.008591674268245697, "Regression_cos_phi": 0.009063780307769775, "Regression_energy": 0.22437454760074615, "Total": 10.26056957244873, "Total_unweighted": 4.567723863758147, "metrics/particle/class_1/efficiency": 0.9066748023033142, "metrics/particle/class_1/pid_accuracy": 0.9777030348777771, "metrics/particle/class_2/efficiency": 0.28153669834136963, "metrics/particle/class_2/pid_accuracy": 0.5580447912216187, "metrics/particle/class_3/efficiency": 0.7074888944625854, "metrics/particle/class_3/pid_accuracy": 0.9731278419494629, "metrics/particle/class_4/efficiency": 0.878892719745636, "metrics/particle/class_4/pid_accuracy": 0.20866142213344574, "metrics/particle/class_5/efficiency": 0.96875, "metrics/particle/class_5/pid_accuracy": 0.4784946143627167, "metrics/particle/count/bias_mean": 0.4880000054836273, "metrics/particle/count/mae": 4.960000038146973, "metrics/particle/count/prediction_mean": 96.73999786376953, "metrics/particle/count/target_mean": 96.25199890136719, "metrics/particle/event/energy_relative_abs_error": 0.060810983180999756, "metrics/particle/event/energy_response_mean": 0.9814264178276062, "metrics/particle/event/met_abs_error": 9.791168212890625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06013373285531998, "metrics/particle/event/scalar_pt_response_mean": 0.9856533408164978, "metrics/particle/event/vector_pt_closure": 0.06383663415908813, "metrics/particle/matched/delta_eta_abs_mean": 0.018944066017866135, "metrics/particle/matched/delta_phi_abs_mean": 0.017796777188777924, "metrics/particle/matched/delta_r_mean": 0.028913017362356186, "metrics/particle/matched/energy_relative_abs_mean": 0.0888742059469223, "metrics/particle/matched/pid_accuracy": 0.949429452419281, "metrics/particle/matched/pt_relative_abs_mean": 0.0847240537405014, "metrics/particle/matching/duplicate_fraction": 0.038453586399555206, "metrics/particle/matching/efficiency": 0.7757552862167358, "metrics/particle/matching/f1": 0.7737937569618225, "metrics/particle/matching/purity": 0.7718420624732971, "metrics/particle/matching/target_coverage_dr0p05": 0.6465943455696106, "metrics/particle/matching/target_coverage_dr0p10": 0.8050949573516846, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6971283555030823, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8350164294242859}, "step_train_time": 0.7578456401824951, "step_valid_time": 1.999780360958539, "step_total_time": 6029.745100736618, "elapsed_training_loop_seconds": 6029.745100736618, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9926528916736544, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16582395652752713, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1470, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1305, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8977807666442502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9081632653061225, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9029421711193778, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09183673469387756, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8776059179556154, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8877551020408163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8826513358133243, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8977807666442502, "step_test_time": 3.1400950169190764, "step_plot_time": 2.2967528000008315, "step_validation_cycle_time": 7.908646175055765} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..7f77928a3ac692c26da05099f44becef9d6d918d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_12mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..5ad33be97a855a252208d496611a48bb19a3cdbd --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:178dac78948f096d16728bd1441bcd8060b20c871566b66ca01df1d8bdd8bd67 +size 4394 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..028fc2b561dbeff5c9cd80fc8a3bfa04248bb774 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..6a6c92640ed73a16edd81ca409f22c865e6695f8 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e0ee99eafc99fccab45e68404cf34573f6e762f1 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..35019f1988b55524ae6c80a867345a8f05abae57 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8c25418cae67b4ccc37d81b7e7c8f8151e7c6faf Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..2ebd5090ed0bdf91baf844bcc31f5a369b9e49a7 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b3092bd953fe497846dde3d72fbbbf55fbb1c53f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..9544d4f7fe02b3690e14c86f6a471326fc1ab501 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d374714a122e072c78356af58ec98ea51ac2bb0c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..a122f1867c6548777f9bd0cdb74dec0cf4303633 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8cb1fae41c84cbf66a610f8e075e8b8591b61d4a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..3e743f475f196e9c4a020eb3d46e568f3a6f91a0 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/plots_step_80000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/runs/train/events.out.tfevents.1794650650.workergpu215.1009066.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/runs/train/events.out.tfevents.1794650650.workergpu215.1009066.0 new file mode 100644 index 0000000000000000000000000000000000000000..fc63f7b03b457119d21f585f22855b7b8f054583 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/runs/train/events.out.tfevents.1794650650.workergpu215.1009066.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29529b353f6c9c8275fac9d54e005780c6d22cd68508b52ff890760fb4e430d9 +size 2953648 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/runs/valid/events.out.tfevents.1794650650.workergpu215.1009066.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/runs/valid/events.out.tfevents.1794650650.workergpu215.1009066.1 new file mode 100644 index 0000000000000000000000000000000000000000..f6310e56554b54a417b2864cfee3dec2ede49fca --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/runs/valid/events.out.tfevents.1794650650.workergpu215.1009066.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7668287a1d2b8c35d3e476c77dc77768df7b5e97bab19aceb888490d9ff5ec32 +size 4026739 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..4a88b6f02e5905daf8a59519e2a6ad6483f5902a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_hcal_12mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_12mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_12mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_12mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd3f1afd7fa0eaf0d025a0520c79a748a15d2746 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_12mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/train.log new file mode 100644 index 0000000000000000000000000000000000000000..9dcd809c43e99562fe2c517fae983ca47e681d14 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/train.log @@ -0,0 +1,1353 @@ +rank0 [2026-11-14 05:04:09,122] INFO : Will use single-gpu: NVIDIA H100 +rank0 [2026-11-14 05:04:09,122] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 05:04:09,351] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 05:04:09,351] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 05:04:09,351] INFO : Instantiating model +rank0 [2026-11-14 05:04:09,369] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 05:04:09,370] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 05:04:09,371] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 05:04:09,371] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 05:04:09,371] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 05:04:09,371] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 05:04:09,371] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,373] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 05:04:09,373] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,375] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 05:04:09,375] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,378] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 05:04:09,378] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,380] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 05:04:09,380] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,383] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 05:04:09,383] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,385] INFO : Backbone initialization took 0.01s +rank0 [2026-11-14 05:04:09,385] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 05:04:09,386] INFO : Initializing output DNNs +rank0 [2026-11-14 05:04:09,386] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,389] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,395] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 05:04:09,395] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 05:04:09,395] INFO : jagged_attention enabled=True +rank0 [2026-11-14 05:04:09,395] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 05:04:09,396] INFO : task_queries enabled=True +rank0 [2026-11-14 05:04:09,396] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 05:04:09,396] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 05:04:09,396] INFO : classification_query parameters: 256 +rank0 [2026-11-14 05:04:09,396] INFO : regression_query parameters: 256 +rank0 [2026-11-14 05:04:09,396] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 05:04:09,396] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 05:04:09,396] INFO : output_mode=elementwise +rank0 [2026-11-14 05:04:09,396] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 05:04:09,396] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 05:04:09,396] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 05:04:09,396] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 05:04:09,396] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 05:04:09,396] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 05:04:09,397] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 05:04:09,397] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 05:04:09,397] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 05:04:09,397] INFO : MLPF __init__ done +rank0 [2026-11-14 05:04:09,397] INFO : Instantiated model +rank0 [2026-11-14 05:04:09,398] INFO : Moving model to device rank=0 +rank0 [2026-11-14 05:04:09,402] INFO : Moved model to device rank=0 +rank0 [2026-11-14 05:04:09,403] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 05:04:09,462] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 05:04:09,462] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 05:04:09,463] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 05:04:09,463] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 05:04:09,464] INFO : Total parameters: 3910930 +rank0 [2026-11-14 05:04:09,465] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 05:04:09,466] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441 +rank0 [2026-11-14 05:04:09,466] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441 +rank0 [2026-11-14 05:04:09,466] INFO : Getting dataloaders +rank0 [2026-11-14 05:04:09,467] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:04:09,617] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,657] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,697] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,737] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,775] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,812] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,860] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,895] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,979] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:10,016] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:10,016] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 05:04:10,017] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 05:04:10,017] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 05:04:10,017] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:04:10,027] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,036] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,045] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,054] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,063] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,072] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,081] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,091] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,099] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,109] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,109] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 05:04:10,109] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 05:04:10,142] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:04:10,316] INFO : Got dataloaders +rank0 [2026-11-14 05:04:10,316] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 05:04:10,316] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 05:04:10,316] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 05:04:10,319] INFO : Creating train_iterator +rank0 [2026-11-14 05:04:10,320] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:10,350] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:04:10,520] INFO : Created train_iterator +rank0 [2026-11-14 05:04:19,288] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.148003125190735, 'Classification': 0.6440470790863038, 'Regression_pt': 0.5139240911602974, 'Regression_eta': 0.9037979310750961, 'Regression_sin_phi': 0.6648890712857246, 'Regression_cos_phi': 0.6466032195091248, 'Regression_energy': 0.6593179452419281}, weights={'Classification_binary': 1.0, 'Classification': 11.098572731018066, 'Regression_pt': 13.908675193786621, 'Regression_eta': 7.908851146697998, 'Regression_sin_phi': 10.750670433044434, 'Regression_cos_phi': 11.05469799041748, 'Regression_energy': 10.841511726379395} +rank0 [2026-11-14 05:04:54,142] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 05:04:54,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:27,283] INFO : Step 1000/500000 rank0 | Train Loss: 12.5896 | Train Loss Unweighted: 5.2980 | Train Loss Components: Classification: 0.0681 | Classification_binary: 4.6273 | Regression_cos_phi: 0.0269 | Regression_energy: 0.2531 | Regression_eta: 0.0397 | Regression_pt: 0.2563 | Regression_sin_phi: 0.0268 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.7557 | Classification_binary: 4.6273 | Regression_cos_phi: 0.2973 | Regression_energy: 2.7436 | Regression_eta: 0.3138 | Regression_pt: 3.5643 | Regression_sin_phi: 0.2878 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7491s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0237 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0833 | max_tensorboard_logging: 0.2184 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2630.3 | uss_MB: 1367.6 | child_rss_MB: 22909.3 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:05:36,220] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 05:05:36,221] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:16,554] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 05:06:16,555] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:39,851] INFO : Step 2000/500000 rank0 | Train Loss: 12.3836 | Train Loss Unweighted: 5.1447 | Train Loss Components: Classification: 0.0440 | Classification_binary: 4.4854 | Regression_cos_phi: 0.0268 | Regression_energy: 0.2679 | Regression_eta: 0.0306 | Regression_pt: 0.2690 | Regression_sin_phi: 0.0209 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4883 | Classification_binary: 4.4854 | Regression_cos_phi: 0.2964 | Regression_energy: 2.9050 | Regression_eta: 0.2422 | Regression_pt: 3.7416 | Regression_sin_phi: 0.2247 | LR: 2.02e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7514s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0844 | max_tensorboard_logging: 0.2161 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2632.2 | uss_MB: 1369.6 | child_rss_MB: 22908.4 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:06:39,853] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:06:58,001] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 05:06:58,001] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:07:38,358] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 05:07:38,358] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:07:52,609] INFO : Step 3000/500000 rank0 | Train Loss: 10.7572 | Train Loss Unweighted: 4.6472 | Train Loss Components: Classification: 0.0446 | Classification_binary: 4.0980 | Regression_cos_phi: 0.0123 | Regression_energy: 0.2292 | Regression_eta: 0.0139 | Regression_pt: 0.2390 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4946 | Classification_binary: 4.0980 | Regression_cos_phi: 0.1361 | Regression_energy: 2.4846 | Regression_eta: 0.1102 | Regression_pt: 3.3236 | Regression_sin_phi: 0.1100 | LR: 2.05e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7446s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0258 | max_optimizer_step: 0.0828 | max_tensorboard_logging: 0.2144 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2634.7 | uss_MB: 1372.4 | child_rss_MB: 22909.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:08:19,836] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 05:08:19,836] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:00,268] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 05:09:00,268] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:05,658] INFO : Step 4000/500000 rank0 | Train Loss: 9.2690 | Train Loss Unweighted: 4.4229 | Train Loss Components: Classification: 0.0429 | Classification_binary: 3.9843 | Regression_cos_phi: 0.0165 | Regression_energy: 0.1680 | Regression_eta: 0.0153 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4756 | Classification_binary: 3.9843 | Regression_cos_phi: 0.1829 | Regression_energy: 1.8219 | Regression_eta: 0.1214 | Regression_pt: 2.5442 | Regression_sin_phi: 0.1388 | LR: 2.08e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7502s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0267 | max_optimizer_step: 0.0834 | max_tensorboard_logging: 0.2149 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2635.2 | uss_MB: 1372.9 | child_rss_MB: 22912.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:09:05,660] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:09:41,771] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 05:09:41,771] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:10:19,690] INFO : Step 5000/500000 rank0 | Train Loss: 10.6934 | Train Loss Unweighted: 4.8094 | Train Loss Components: Classification: 0.0409 | Classification_binary: 4.2799 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2238 | Regression_eta: 0.0135 | Regression_pt: 0.2285 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4538 | Classification_binary: 4.2799 | Regression_cos_phi: 0.1174 | Regression_energy: 2.4261 | Regression_eta: 0.1067 | Regression_pt: 3.1778 | Regression_sin_phi: 0.1318 | LR: 2.13e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7881s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0269 | max_optimizer_step: 0.0837 | max_tensorboard_logging: 0.2154 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2634.6 | uss_MB: 1372.3 | child_rss_MB: 22911.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:10:23,896] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 05:10:23,896] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:04,114] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 05:11:04,115] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:32,497] INFO : Step 6000/500000 rank0 | Train Loss: 8.7115 | Train Loss Unweighted: 4.2087 | Train Loss Components: Classification: 0.0417 | Classification_binary: 3.8025 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1685 | Regression_eta: 0.0085 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4628 | Classification_binary: 3.8025 | Regression_cos_phi: 0.1150 | Regression_energy: 1.8265 | Regression_eta: 0.0671 | Regression_pt: 2.3489 | Regression_sin_phi: 0.0887 | LR: 2.19e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7392s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0837 | max_tensorboard_logging: 0.2130 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2634.6 | uss_MB: 1372.3 | child_rss_MB: 22909.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:11:32,499] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:11:45,871] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 05:11:45,872] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:27,063] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 05:12:27,064] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:46,439] INFO : Step 7000/500000 rank0 | Train Loss: 10.3344 | Train Loss Unweighted: 4.4695 | Train Loss Components: Classification: 0.0342 | Classification_binary: 3.9430 | Regression_cos_phi: 0.0137 | Regression_energy: 0.2263 | Regression_eta: 0.0121 | Regression_pt: 0.2310 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3792 | Classification_binary: 3.9430 | Regression_cos_phi: 0.1512 | Regression_energy: 2.4537 | Regression_eta: 0.0956 | Regression_pt: 3.2129 | Regression_sin_phi: 0.0989 | LR: 2.26e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7569s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0278 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.2123 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2665.3 | uss_MB: 1403.0 | child_rss_MB: 22914.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:13:09,121] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 05:13:09,122] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:49,539] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 05:13:49,539] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:59,919] INFO : Step 8000/500000 rank0 | Train Loss: 8.6836 | Train Loss Unweighted: 4.4360 | Train Loss Components: Classification: 0.0417 | Classification_binary: 4.0501 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1536 | Regression_eta: 0.0141 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4628 | Classification_binary: 4.0501 | Regression_cos_phi: 0.1104 | Regression_energy: 1.6654 | Regression_eta: 0.1118 | Regression_pt: 2.1719 | Regression_sin_phi: 0.1112 | LR: 2.34e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7752s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.1116 | max_tensorboard_logging: 0.2125 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2665.4 | uss_MB: 1403.1 | child_rss_MB: 22910.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:13:59,921] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:13:59,967] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:14:31,710] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 05:14:31,710] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:13,105] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 05:15:13,105] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:14,365] INFO : Step 9000/500000 rank0 | Train Loss: 8.4548 | Train Loss Unweighted: 4.1895 | Train Loss Components: Classification: 0.0327 | Classification_binary: 3.8016 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1543 | Regression_eta: 0.0184 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3633 | Classification_binary: 3.8016 | Regression_cos_phi: 0.1153 | Regression_energy: 1.6725 | Regression_eta: 0.1455 | Regression_pt: 2.2337 | Regression_sin_phi: 0.1228 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.7438s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0267 | max_optimizer_step: 0.0833 | max_tensorboard_logging: 0.2200 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2634.0 | uss_MB: 1371.8 | child_rss_MB: 22909.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:15:55,171] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 05:15:55,171] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:29,023] INFO : Step 10000/500000 rank0 | Train Loss: 8.9702 | Train Loss Unweighted: 4.4252 | Train Loss Components: Classification: 0.0385 | Classification_binary: 4.0157 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1678 | Regression_eta: 0.0083 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0124 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4272 | Classification_binary: 4.0157 | Regression_cos_phi: 0.1103 | Regression_energy: 1.8197 | Regression_eta: 0.0656 | Regression_pt: 2.3988 | Regression_sin_phi: 0.1328 | LR: 2.52e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7887s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0833 | max_tensorboard_logging: 0.2527 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2634.5 | uss_MB: 1372.2 | child_rss_MB: 22910.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:16:29,025] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:16:29,072] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:16:29,073] INFO : Running validation on rank0 +rank0 [2026-11-14 05:16:29,073] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:16:29,073] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:16:29,199] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 05:16:29,314] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:16:29,371] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:16:29,411] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:16:29,450] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:16:29,483] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:16:29,522] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:16:29,563] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:16:29,599] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:16:29,654] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:16:29,708] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:16:29,746] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:16:29,786] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:16:29,819] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:16:29,859] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:16:29,897] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:16:29,935] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:16:29,938] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:16:29,939] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:16:29,941] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:16:29,943] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:16:29,944] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:16:29,946] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:16:29,947] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:16:29,949] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:16:29,951] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:16:29,952] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:16:29,954] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:16:29,955] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:16:29,957] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:16:29,958] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:16:29,960] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:16:30,981] INFO : VALIDATION | Step=10000/500000 | Train Loss=8.9702 | Valid Loss=10.1504 | Train Loss Unweighted=4.4252 | Valid Loss Unweighted=4.5026 | Stale=0 | ETA=604.9m +rank0 [2026-11-14 05:16:30,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:16:31,080] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:16:31,088] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:16:31,137] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:16:31,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:16:31,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:16:31,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:16:32,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:16:32,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:16:32,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:16:32,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:16:32,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:16:32,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:16:33,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:16:33,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:16:33,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:16:33,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:16:33,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:16:33,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:16:34,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:16:34,096] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:16:34,114] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:16:34,500] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:16:34,805] INFO : Plotted number of elements +rank0 [2026-11-14 05:16:35,757] INFO : Plotted jets +rank0 [2026-11-14 05:16:36,465] INFO : Plotted jet ratio +rank0 [2026-11-14 05:16:45,473] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 05:16:45,474] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:27,081] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 05:17:27,082] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:51,628] INFO : Step 11000/500000 rank0 | Train Loss: 10.4528 | Train Loss Unweighted: 4.5009 | Train Loss Components: Classification: 0.0314 | Classification_binary: 3.9681 | Regression_cos_phi: 0.0086 | Regression_energy: 0.2310 | Regression_eta: 0.0130 | Regression_pt: 0.2405 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3486 | Classification_binary: 3.9681 | Regression_cos_phi: 0.0948 | Regression_energy: 2.5039 | Regression_eta: 0.1026 | Regression_pt: 3.3453 | Regression_sin_phi: 0.0895 | LR: 2.63e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7669s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.2170 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2817.6 | uss_MB: 1563.5 | child_rss_MB: 22913.9 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 05:18:09,432] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 05:18:09,433] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:51,213] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 05:18:51,213] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:06,729] INFO : Step 12000/500000 rank0 | Train Loss: 10.4375 | Train Loss Unweighted: 4.3321 | Train Loss Components: Classification: 0.0360 | Classification_binary: 3.7863 | Regression_cos_phi: 0.0090 | Regression_energy: 0.2385 | Regression_eta: 0.0085 | Regression_pt: 0.2441 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3996 | Classification_binary: 3.7863 | Regression_cos_phi: 0.0997 | Regression_energy: 2.5855 | Regression_eta: 0.0670 | Regression_pt: 3.3952 | Regression_sin_phi: 0.1040 | LR: 2.75e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8108s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2589 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2817.6 | uss_MB: 1563.5 | child_rss_MB: 22915.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:19:06,731] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:19:06,778] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:19:33,925] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 05:19:33,925] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:15,160] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 05:20:15,160] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:21,313] INFO : Step 13000/500000 rank0 | Train Loss: 9.2974 | Train Loss Unweighted: 4.3393 | Train Loss Components: Classification: 0.0370 | Classification_binary: 3.8907 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1865 | Regression_eta: 0.0155 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4102 | Classification_binary: 3.8907 | Regression_cos_phi: 0.1031 | Regression_energy: 2.0221 | Regression_eta: 0.1225 | Regression_pt: 2.6184 | Regression_sin_phi: 0.1303 | LR: 2.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7612s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0847 | max_tensorboard_logging: 0.2187 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2745.1 | uss_MB: 1491.0 | child_rss_MB: 22914.7 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:20:57,085] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:20:57,085] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:35,330] INFO : Step 14000/500000 rank0 | Train Loss: 9.2849 | Train Loss Unweighted: 4.1741 | Train Loss Components: Classification: 0.0400 | Classification_binary: 3.7166 | Regression_cos_phi: 0.0043 | Regression_energy: 0.2015 | Regression_eta: 0.0060 | Regression_pt: 0.2005 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4444 | Classification_binary: 3.7166 | Regression_cos_phi: 0.0470 | Regression_energy: 2.1846 | Regression_eta: 0.0476 | Regression_pt: 2.7882 | Regression_sin_phi: 0.0566 | LR: 3.02e-05 | DataLoad Time: 0.0024s | Model Forward Time: 0.7628s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0886 | max_tensorboard_logging: 0.2137 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2745.4 | uss_MB: 1491.3 | child_rss_MB: 22916.3 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:21:35,332] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:21:35,378] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:21:39,018] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 05:21:39,018] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:20,071] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 05:22:20,071] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:49,803] INFO : Step 15000/500000 rank0 | Train Loss: 8.9437 | Train Loss Unweighted: 4.3029 | Train Loss Components: Classification: 0.0412 | Classification_binary: 3.8822 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1724 | Regression_eta: 0.0153 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4577 | Classification_binary: 3.8822 | Regression_cos_phi: 0.0846 | Regression_energy: 1.8691 | Regression_eta: 0.1211 | Regression_pt: 2.4228 | Regression_sin_phi: 0.1061 | LR: 3.17e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7621s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0252 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2156 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2748.3 | uss_MB: 1494.2 | child_rss_MB: 22917.3 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:23:02,684] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 05:23:02,684] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:23:43,379] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 05:23:43,379] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:03,342] INFO : Step 16000/500000 rank0 | Train Loss: 9.6332 | Train Loss Unweighted: 4.4402 | Train Loss Components: Classification: 0.0353 | Classification_binary: 3.9738 | Regression_cos_phi: 0.0117 | Regression_energy: 0.1946 | Regression_eta: 0.0084 | Regression_pt: 0.2013 | Regression_sin_phi: 0.0151 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3919 | Classification_binary: 3.9738 | Regression_cos_phi: 0.1293 | Regression_energy: 2.1095 | Regression_eta: 0.0664 | Regression_pt: 2.8004 | Regression_sin_phi: 0.1619 | LR: 3.33e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7652s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0250 | max_loss: 0.0024 | max_backward: 0.0276 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.2134 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2748.8 | uss_MB: 1494.7 | child_rss_MB: 22916.6 | allocated_MB: 126.1 | reserved_MB: 632.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:24:03,343] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:24:03,389] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:24:25,363] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 05:24:25,363] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:06,794] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 05:25:06,794] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:17,735] INFO : Step 17000/500000 rank0 | Train Loss: 10.2397 | Train Loss Unweighted: 4.1611 | Train Loss Components: Classification: 0.0404 | Classification_binary: 3.6110 | Regression_cos_phi: 0.0173 | Regression_energy: 0.2304 | Regression_eta: 0.0179 | Regression_pt: 0.2298 | Regression_sin_phi: 0.0142 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4486 | Classification_binary: 3.6110 | Regression_cos_phi: 0.1909 | Regression_energy: 2.4977 | Regression_eta: 0.1419 | Regression_pt: 3.1968 | Regression_sin_phi: 0.1529 | LR: 3.51e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8078s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2744.5 | uss_MB: 1490.4 | child_rss_MB: 22914.9 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:25:49,409] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 05:25:49,409] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:31,269] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 05:26:31,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:33,151] INFO : Step 18000/500000 rank0 | Train Loss: 9.1121 | Train Loss Unweighted: 4.1689 | Train Loss Components: Classification: 0.0355 | Classification_binary: 3.7259 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1896 | Regression_eta: 0.0065 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3939 | Classification_binary: 3.7259 | Regression_cos_phi: 0.1095 | Regression_energy: 2.0560 | Regression_eta: 0.0516 | Regression_pt: 2.6813 | Regression_sin_phi: 0.0939 | LR: 3.69e-05 | DataLoad Time: 0.0004s | Model Forward Time: 0.7652s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0269 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.2173 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2745.5 | uss_MB: 1491.4 | child_rss_MB: 22915.4 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:26:33,153] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:26:33,201] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:27:13,839] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 05:27:13,840] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:27:48,253] INFO : Step 19000/500000 rank0 | Train Loss: 7.8044 | Train Loss Unweighted: 3.9318 | Train Loss Components: Classification: 0.0372 | Classification_binary: 3.5818 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1433 | Regression_eta: 0.0094 | Regression_pt: 0.1452 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4132 | Classification_binary: 3.5818 | Regression_cos_phi: 0.0826 | Regression_energy: 1.5540 | Regression_eta: 0.0747 | Regression_pt: 2.0191 | Regression_sin_phi: 0.0790 | LR: 3.88e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7555s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0273 | max_optimizer_step: 0.0851 | max_tensorboard_logging: 0.2154 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2748.4 | uss_MB: 1494.3 | child_rss_MB: 22914.4 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:27:56,254] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 05:27:56,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:37,545] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 05:28:37,545] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:29:02,927] INFO : Step 20000/500000 rank0 | Train Loss: 10.1618 | Train Loss Unweighted: 4.3767 | Train Loss Components: Classification: 0.0400 | Classification_binary: 3.8541 | Regression_cos_phi: 0.0090 | Regression_energy: 0.2319 | Regression_eta: 0.0137 | Regression_pt: 0.2188 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4435 | Classification_binary: 3.8541 | Regression_cos_phi: 0.0990 | Regression_energy: 2.5139 | Regression_eta: 0.1086 | Regression_pt: 3.0426 | Regression_sin_phi: 0.1001 | LR: 4.07e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7576s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.2124 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2748.7 | uss_MB: 1494.6 | child_rss_MB: 22916.6 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:29:02,928] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:29:02,977] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:29:02,978] INFO : Running validation on rank0 +rank0 [2026-11-14 05:29:02,978] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:29:02,978] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:29:03,079] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 05:29:03,161] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:29:03,239] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:29:03,289] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:29:03,335] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:29:03,381] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:29:03,428] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:29:03,475] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:29:03,552] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:29:03,629] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:29:03,706] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:29:03,754] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:29:03,799] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:29:03,845] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:29:03,892] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:29:03,938] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:29:04,004] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:29:04,008] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:29:04,010] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:29:04,012] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:29:04,014] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:29:04,015] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:29:04,017] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:29:04,019] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:29:04,020] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:29:04,022] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:29:04,023] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:29:04,025] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:29:04,027] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:29:04,028] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:29:04,030] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:29:04,032] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:29:05,009] INFO : VALIDATION | Step=20000/500000 | Train Loss=10.1618 | Valid Loss=9.6373 | Train Loss Unweighted=4.3767 | Valid Loss Unweighted=4.3313 | Stale=0 | ETA=597.9m +rank0 [2026-11-14 05:29:05,009] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:29:05,127] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:29:05,137] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:29:05,204] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:29:05,714] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:29:05,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:29:06,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:29:06,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:29:06,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:29:06,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:29:06,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:29:06,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:29:07,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:29:07,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:29:07,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:29:07,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:29:07,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:29:07,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:29:07,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:29:08,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:29:08,218] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:29:08,226] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:29:08,610] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:29:08,846] INFO : Plotted number of elements +rank0 [2026-11-14 05:29:09,778] INFO : Plotted jets +rank0 [2026-11-14 05:29:10,475] INFO : Plotted jet ratio +rank0 [2026-11-14 05:29:27,977] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 05:29:27,978] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:09,144] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 05:30:09,144] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:24,566] INFO : Step 21000/500000 rank0 | Train Loss: 7.6211 | Train Loss Unweighted: 3.8977 | Train Loss Components: Classification: 0.0367 | Classification_binary: 3.5622 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1387 | Regression_eta: 0.0065 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4076 | Classification_binary: 3.5622 | Regression_cos_phi: 0.0708 | Regression_energy: 1.5037 | Regression_eta: 0.0511 | Regression_pt: 1.9535 | Regression_sin_phi: 0.0723 | LR: 4.28e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7733s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0024 | max_backward: 0.0260 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2248 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2777.5 | uss_MB: 1523.5 | child_rss_MB: 22916.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:30:50,042] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 05:30:50,042] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:31:30,285] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 05:31:30,285] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:31:36,899] INFO : Step 22000/500000 rank0 | Train Loss: 9.1272 | Train Loss Unweighted: 4.1516 | Train Loss Components: Classification: 0.0302 | Classification_binary: 3.7008 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1909 | Regression_eta: 0.0195 | Regression_pt: 0.1916 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3355 | Classification_binary: 3.7008 | Regression_cos_phi: 0.0926 | Regression_energy: 2.0695 | Regression_eta: 0.1541 | Regression_pt: 2.6648 | Regression_sin_phi: 0.1099 | LR: 4.50e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7676s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0258 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.2206 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2778.6 | uss_MB: 1524.6 | child_rss_MB: 22916.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:31:36,901] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:31:36,949] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:32:12,562] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 05:32:12,562] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:52,121] INFO : Step 23000/500000 rank0 | Train Loss: 8.7751 | Train Loss Unweighted: 4.1982 | Train Loss Components: Classification: 0.0311 | Classification_binary: 3.7870 | Regression_cos_phi: 0.0144 | Regression_energy: 0.1750 | Regression_eta: 0.0049 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3457 | Classification_binary: 3.7870 | Regression_cos_phi: 0.1593 | Regression_energy: 1.8970 | Regression_eta: 0.0390 | Regression_pt: 2.4247 | Regression_sin_phi: 0.1224 | LR: 4.73e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7952s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0024 | max_backward: 0.0270 | max_optimizer_step: 0.1175 | max_tensorboard_logging: 0.2160 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2780.6 | uss_MB: 1526.6 | child_rss_MB: 22916.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:32:55,222] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 05:32:55,222] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:33:37,147] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 05:33:37,147] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:34:07,293] INFO : Step 24000/500000 rank0 | Train Loss: 8.4818 | Train Loss Unweighted: 4.0741 | Train Loss Components: Classification: 0.0372 | Classification_binary: 3.6757 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1693 | Regression_eta: 0.0083 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4129 | Classification_binary: 3.6757 | Regression_cos_phi: 0.1075 | Regression_energy: 1.8355 | Regression_eta: 0.0654 | Regression_pt: 2.2685 | Regression_sin_phi: 0.1162 | LR: 4.97e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8176s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0247 | max_loss: 0.0023 | max_backward: 0.0274 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2780.6 | uss_MB: 1526.6 | child_rss_MB: 22916.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:34:07,295] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:34:07,343] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:34:19,669] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 05:34:19,669] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:00,956] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 05:35:00,957] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:22,068] INFO : Step 25000/500000 rank0 | Train Loss: 8.1232 | Train Loss Unweighted: 3.9974 | Train Loss Components: Classification: 0.0371 | Classification_binary: 3.6264 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1551 | Regression_eta: 0.0045 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4112 | Classification_binary: 3.6264 | Regression_cos_phi: 0.1126 | Regression_energy: 1.6814 | Regression_eta: 0.0359 | Regression_pt: 2.1673 | Regression_sin_phi: 0.0881 | LR: 5.22e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7750s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0299 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2188 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2777.5 | uss_MB: 1523.6 | child_rss_MB: 22919.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 32.3 +rank0 [2026-11-14 05:35:43,577] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 05:35:43,577] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:36:25,387] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 05:36:25,388] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:36:37,127] INFO : Step 26000/500000 rank0 | Train Loss: 9.4706 | Train Loss Unweighted: 4.2128 | Train Loss Components: Classification: 0.0345 | Classification_binary: 3.7432 | Regression_cos_phi: 0.0064 | Regression_energy: 0.2042 | Regression_eta: 0.0067 | Regression_pt: 0.2108 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3826 | Classification_binary: 3.7432 | Regression_cos_phi: 0.0710 | Regression_energy: 2.2143 | Regression_eta: 0.0532 | Regression_pt: 2.9318 | Regression_sin_phi: 0.0745 | LR: 5.47e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7440s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.2105 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2778.7 | uss_MB: 1524.7 | child_rss_MB: 22917.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:36:37,129] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:36:37,176] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:37:07,790] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 05:37:07,790] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:49,081] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 05:37:49,081] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:51,561] INFO : Step 27000/500000 rank0 | Train Loss: 9.4683 | Train Loss Unweighted: 4.1193 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.6410 | Regression_cos_phi: 0.0113 | Regression_energy: 0.2089 | Regression_eta: 0.0074 | Regression_pt: 0.2131 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3342 | Classification_binary: 3.6410 | Regression_cos_phi: 0.1250 | Regression_energy: 2.2653 | Regression_eta: 0.0585 | Regression_pt: 2.9633 | Regression_sin_phi: 0.0811 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7691s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0033 | max_backward: 0.0273 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.2157 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.1 | uss_MB: 1527.2 | child_rss_MB: 22917.0 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:38:31,732] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 05:38:31,732] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:06,212] INFO : Step 28000/500000 rank0 | Train Loss: 8.9539 | Train Loss Unweighted: 4.0407 | Train Loss Components: Classification: 0.0323 | Classification_binary: 3.5991 | Regression_cos_phi: 0.0123 | Regression_energy: 0.1852 | Regression_eta: 0.0101 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3586 | Classification_binary: 3.5991 | Regression_cos_phi: 0.1357 | Regression_energy: 2.0079 | Regression_eta: 0.0797 | Regression_pt: 2.6618 | Regression_sin_phi: 0.1111 | LR: 6.01e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8085s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0025 | max_backward: 0.0260 | max_optimizer_step: 0.0854 | max_tensorboard_logging: 0.2643 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.2 | uss_MB: 1527.3 | child_rss_MB: 22918.4 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:39:06,214] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:39:06,261] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:39:13,335] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 05:39:13,335] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:55,100] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 05:39:55,100] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:40:20,924] INFO : Step 29000/500000 rank0 | Train Loss: 9.4892 | Train Loss Unweighted: 4.0655 | Train Loss Components: Classification: 0.0328 | Classification_binary: 3.5802 | Regression_cos_phi: 0.0082 | Regression_energy: 0.2089 | Regression_eta: 0.0103 | Regression_pt: 0.2177 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3639 | Classification_binary: 3.5802 | Regression_cos_phi: 0.0902 | Regression_energy: 2.2643 | Regression_eta: 0.0815 | Regression_pt: 3.0286 | Regression_sin_phi: 0.0805 | LR: 6.29e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7792s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0025 | max_backward: 0.0269 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2155 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2781.5 | uss_MB: 1527.5 | child_rss_MB: 22918.1 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:40:37,517] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 05:40:37,517] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:19,220] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 05:41:19,220] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:36,195] INFO : Step 30000/500000 rank0 | Train Loss: 8.0177 | Train Loss Unweighted: 3.7923 | Train Loss Components: Classification: 0.0289 | Classification_binary: 3.4128 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1638 | Regression_eta: 0.0079 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3203 | Classification_binary: 3.4128 | Regression_cos_phi: 0.0663 | Regression_energy: 1.7761 | Regression_eta: 0.0624 | Regression_pt: 2.2912 | Regression_sin_phi: 0.0886 | LR: 6.58e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7774s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0273 | max_loss: 0.0023 | max_backward: 0.0274 | max_optimizer_step: 0.0871 | max_tensorboard_logging: 0.2234 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2778.6 | uss_MB: 1524.6 | child_rss_MB: 22921.7 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:41:36,197] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:41:36,245] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:41:36,246] INFO : Running validation on rank0 +rank0 [2026-11-14 05:41:36,246] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:41:36,246] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:41:36,349] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 05:41:36,430] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:41:36,509] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:41:36,557] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:41:36,602] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:41:36,648] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:41:36,694] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:41:36,740] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:41:36,804] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:41:36,882] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:41:36,959] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:41:37,005] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:41:37,050] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:41:37,097] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:41:37,145] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:41:37,192] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:41:37,258] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:41:37,262] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:41:37,264] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:41:37,266] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:41:37,267] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:41:37,269] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:41:37,270] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:41:37,272] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:41:37,274] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:41:37,275] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:41:37,277] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:41:37,278] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:41:37,280] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:41:37,281] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:41:37,283] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:41:37,285] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:41:38,268] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.0177 | Valid Loss=9.3713 | Train Loss Unweighted=3.7923 | Valid Loss Unweighted=4.1966 | Stale=0 | ETA=587.0m +rank0 [2026-11-14 05:41:38,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:41:38,365] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:41:38,375] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:41:38,426] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:41:38,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:41:39,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:41:39,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:41:39,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:41:39,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:41:39,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:41:39,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:41:40,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:41:40,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:41:40,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:41:40,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:41:40,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:41:40,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:41:40,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:41:41,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:41:41,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:41:41,420] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:41:41,427] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:41:41,820] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:41:42,059] INFO : Plotted number of elements +rank0 [2026-11-14 05:41:42,984] INFO : Plotted jets +rank0 [2026-11-14 05:41:43,684] INFO : Plotted jet ratio +rank0 [2026-11-14 05:42:10,080] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 05:42:10,080] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:42:51,933] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 05:42:51,933] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:42:59,329] INFO : Step 31000/500000 rank0 | Train Loss: 9.2547 | Train Loss Unweighted: 4.0277 | Train Loss Components: Classification: 0.0323 | Classification_binary: 3.5584 | Regression_cos_phi: 0.0104 | Regression_energy: 0.2013 | Regression_eta: 0.0082 | Regression_pt: 0.2031 | Regression_sin_phi: 0.0140 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3582 | Classification_binary: 3.5584 | Regression_cos_phi: 0.1150 | Regression_energy: 2.1821 | Regression_eta: 0.0652 | Regression_pt: 2.8248 | Regression_sin_phi: 0.1510 | LR: 6.88e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7772s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.2215 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2906.9 | uss_MB: 1653.0 | child_rss_MB: 22916.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:43:34,365] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 05:43:34,366] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:14,417] INFO : Step 32000/500000 rank0 | Train Loss: 10.3134 | Train Loss Unweighted: 4.3098 | Train Loss Components: Classification: 0.0343 | Classification_binary: 3.7697 | Regression_cos_phi: 0.0199 | Regression_energy: 0.2235 | Regression_eta: 0.0135 | Regression_pt: 0.2334 | Regression_sin_phi: 0.0156 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3811 | Classification_binary: 3.7697 | Regression_cos_phi: 0.2195 | Regression_energy: 2.4232 | Regression_eta: 0.1065 | Regression_pt: 3.2461 | Regression_sin_phi: 0.1673 | LR: 7.19e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8172s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0246 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0878 | max_tensorboard_logging: 0.2232 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2907.0 | uss_MB: 1653.0 | child_rss_MB: 22916.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:44:14,419] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:44:14,467] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:44:17,004] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 05:44:17,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:59,120] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 05:44:59,121] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:45:29,840] INFO : Step 33000/500000 rank0 | Train Loss: 7.5124 | Train Loss Unweighted: 3.6857 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.3412 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1491 | Regression_eta: 0.0082 | Regression_pt: 0.1478 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2821 | Classification_binary: 3.3412 | Regression_cos_phi: 0.0841 | Regression_energy: 1.6163 | Regression_eta: 0.0646 | Regression_pt: 2.0562 | Regression_sin_phi: 0.0680 | LR: 7.51e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7902s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0247 | max_loss: 0.0025 | max_backward: 0.0272 | max_optimizer_step: 0.0875 | max_tensorboard_logging: 0.2213 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2803.7 | uss_MB: 1549.7 | child_rss_MB: 22916.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:45:41,579] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:45:41,579] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:23,313] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:46:23,313] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:44,788] INFO : Step 34000/500000 rank0 | Train Loss: 8.2648 | Train Loss Unweighted: 3.9463 | Train Loss Components: Classification: 0.0328 | Classification_binary: 3.5580 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1630 | Regression_eta: 0.0078 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3643 | Classification_binary: 3.5580 | Regression_cos_phi: 0.0986 | Regression_energy: 1.7673 | Regression_eta: 0.0613 | Regression_pt: 2.3144 | Regression_sin_phi: 0.1009 | LR: 7.83e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7770s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2216 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2804.4 | uss_MB: 1550.5 | child_rss_MB: 22917.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:46:44,790] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:46:44,837] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:47:05,824] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:47:05,825] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:47:47,598] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:47:47,598] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:00,210] INFO : Step 35000/500000 rank0 | Train Loss: 7.6021 | Train Loss Unweighted: 3.7834 | Train Loss Components: Classification: 0.0355 | Classification_binary: 3.4405 | Regression_cos_phi: 0.0044 | Regression_energy: 0.1451 | Regression_eta: 0.0046 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3941 | Classification_binary: 3.4405 | Regression_cos_phi: 0.0481 | Regression_energy: 1.5736 | Regression_eta: 0.0364 | Regression_pt: 2.0335 | Regression_sin_phi: 0.0758 | LR: 8.16e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7628s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0024 | max_backward: 0.0272 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2139 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.7 | uss_MB: 1552.8 | child_rss_MB: 22916.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:48:30,345] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:48:30,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:49:12,077] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:49:12,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:49:15,137] INFO : Step 36000/500000 rank0 | Train Loss: 9.2753 | Train Loss Unweighted: 4.0148 | Train Loss Components: Classification: 0.0296 | Classification_binary: 3.5439 | Regression_cos_phi: 0.0049 | Regression_energy: 0.2049 | Regression_eta: 0.0104 | Regression_pt: 0.2113 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3289 | Classification_binary: 3.5439 | Regression_cos_phi: 0.0542 | Regression_energy: 2.2209 | Regression_eta: 0.0821 | Regression_pt: 2.9395 | Regression_sin_phi: 0.1057 | LR: 8.50e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7657s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.2117 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.8 | uss_MB: 1552.9 | child_rss_MB: 22915.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:49:15,138] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:49:15,185] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:49:54,536] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:49:54,536] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:30,036] INFO : Step 37000/500000 rank0 | Train Loss: 8.2987 | Train Loss Unweighted: 3.8319 | Train Loss Components: Classification: 0.0336 | Classification_binary: 3.4297 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1728 | Regression_eta: 0.0079 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3729 | Classification_binary: 3.4297 | Regression_cos_phi: 0.0983 | Regression_energy: 1.8733 | Regression_eta: 0.0626 | Regression_pt: 2.3680 | Regression_sin_phi: 0.0939 | LR: 8.85e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7736s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0278 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2183 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.8 | uss_MB: 1552.9 | child_rss_MB: 22917.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:50:36,833] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:50:36,833] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:18,789] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:51:18,789] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:45,241] INFO : Step 38000/500000 rank0 | Train Loss: 7.6105 | Train Loss Unweighted: 3.7278 | Train Loss Components: Classification: 0.0366 | Classification_binary: 3.3744 | Regression_cos_phi: 0.0120 | Regression_energy: 0.1404 | Regression_eta: 0.0115 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4057 | Classification_binary: 3.3744 | Regression_cos_phi: 0.1330 | Regression_energy: 1.5224 | Regression_eta: 0.0910 | Regression_pt: 1.9376 | Regression_sin_phi: 0.1463 | LR: 9.21e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7730s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0023 | max_backward: 0.0274 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.2184 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2804.5 | uss_MB: 1550.6 | child_rss_MB: 22918.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:51:45,242] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:51:45,288] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:52:01,381] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:52:01,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:43,173] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:52:43,173] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:59,999] INFO : Step 39000/500000 rank0 | Train Loss: 7.7061 | Train Loss Unweighted: 3.6005 | Train Loss Components: Classification: 0.0296 | Classification_binary: 3.2299 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1547 | Regression_eta: 0.0108 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3280 | Classification_binary: 3.2299 | Regression_cos_phi: 0.1131 | Regression_energy: 1.6770 | Regression_eta: 0.0853 | Regression_pt: 2.1798 | Regression_sin_phi: 0.0930 | LR: 9.57e-05 | DataLoad Time: 0.0013s | Model Forward Time: 0.7704s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0276 | max_optimizer_step: 0.0863 | max_tensorboard_logging: 0.2201 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2806.9 | uss_MB: 1553.0 | child_rss_MB: 22918.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:53:24,752] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:53:24,752] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:05,795] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:54:05,795] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:13,537] INFO : Step 40000/500000 rank0 | Train Loss: 7.9346 | Train Loss Unweighted: 3.7324 | Train Loss Components: Classification: 0.0317 | Classification_binary: 3.3526 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1576 | Regression_eta: 0.0104 | Regression_pt: 0.1584 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3515 | Classification_binary: 3.3526 | Regression_cos_phi: 0.0920 | Regression_energy: 1.7088 | Regression_eta: 0.0821 | Regression_pt: 2.2035 | Regression_sin_phi: 0.1440 | LR: 9.94e-05 | DataLoad Time: 0.0093s | Model Forward Time: 0.7691s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0275 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2170 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2807.3 | uss_MB: 1553.4 | child_rss_MB: 22918.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:54:13,539] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:54:13,587] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:54:13,588] INFO : Running validation on rank0 +rank0 [2026-11-14 05:54:13,588] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:54:13,588] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:54:13,692] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:54:13,776] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:54:13,854] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:54:13,905] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:54:13,951] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:54:13,996] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:54:14,042] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:54:14,088] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:54:14,165] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:54:14,245] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:54:14,323] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:54:14,370] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:54:14,417] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:54:14,463] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:54:14,510] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:54:14,556] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:54:14,634] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:54:14,640] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:54:14,641] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:54:14,643] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:54:14,645] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:54:14,646] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:54:14,648] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:54:14,649] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:54:14,651] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:54:14,653] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:54:14,654] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:54:14,656] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:54:14,657] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:54:14,659] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:54:14,661] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:54:14,662] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:54:15,648] INFO : VALIDATION | Step=40000/500000 | Train Loss=7.9346 | Valid Loss=8.8835 | Train Loss Unweighted=3.7324 | Valid Loss Unweighted=3.9793 | Stale=0 | ETA=576.0m +rank0 [2026-11-14 05:54:15,648] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:54:15,750] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:54:15,759] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:54:15,810] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:54:16,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:54:16,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:54:16,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:54:16,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:54:16,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:54:17,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:54:17,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:54:17,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:54:17,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:54:17,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:54:17,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:54:18,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:54:18,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:54:18,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:54:18,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:54:18,687] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:54:18,791] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:54:18,798] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:54:19,173] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:54:19,411] INFO : Plotted number of elements +rank0 [2026-11-14 05:54:20,349] INFO : Plotted jets +rank0 [2026-11-14 05:54:21,044] INFO : Plotted jet ratio +rank0 [2026-11-14 05:54:55,797] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:54:55,797] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:55:36,043] INFO : Step 41000/500000 rank0 | Train Loss: 7.2046 | Train Loss Unweighted: 3.5449 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.2137 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1318 | Regression_eta: 0.0111 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3592 | Classification_binary: 3.2137 | Regression_cos_phi: 0.0921 | Regression_energy: 1.4293 | Regression_eta: 0.0878 | Regression_pt: 1.9203 | Regression_sin_phi: 0.1022 | LR: 1.03e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.8240s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.1345 | max_tensorboard_logging: 0.2189 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2828.7 | uss_MB: 1574.8 | child_rss_MB: 22919.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:55:37,963] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:55:37,963] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:19,323] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:56:19,323] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:50,502] INFO : Step 42000/500000 rank0 | Train Loss: 7.6894 | Train Loss Unweighted: 3.7740 | Train Loss Components: Classification: 0.0331 | Classification_binary: 3.4203 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1418 | Regression_eta: 0.0069 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0173 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3675 | Classification_binary: 3.4203 | Regression_cos_phi: 0.1015 | Regression_energy: 1.5374 | Regression_eta: 0.0548 | Regression_pt: 2.0222 | Regression_sin_phi: 0.1857 | LR: 1.07e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7589s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0851 | max_tensorboard_logging: 0.2135 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2829.4 | uss_MB: 1575.6 | child_rss_MB: 22917.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:56:50,505] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:56:50,552] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:57:01,538] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:57:01,538] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:57:43,057] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:57:43,057] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:04,933] INFO : Step 43000/500000 rank0 | Train Loss: 7.5353 | Train Loss Unweighted: 3.7548 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.4150 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1415 | Regression_eta: 0.0070 | Regression_pt: 0.1463 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2906 | Classification_binary: 3.4150 | Regression_cos_phi: 0.0990 | Regression_energy: 1.5345 | Regression_eta: 0.0554 | Regression_pt: 2.0344 | Regression_sin_phi: 0.1064 | LR: 1.11e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7540s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0260 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2066 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2831.5 | uss_MB: 1577.7 | child_rss_MB: 22919.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:58:24,654] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:58:24,654] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:59:06,165] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:59:06,166] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:59:19,057] INFO : Step 44000/500000 rank0 | Train Loss: 7.9219 | Train Loss Unweighted: 3.6805 | Train Loss Components: Classification: 0.0352 | Classification_binary: 3.2998 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1627 | Regression_eta: 0.0049 | Regression_pt: 0.1629 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3910 | Classification_binary: 3.2998 | Regression_cos_phi: 0.0826 | Regression_energy: 1.7637 | Regression_eta: 0.0389 | Regression_pt: 2.2655 | Regression_sin_phi: 0.0803 | LR: 1.15e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7726s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2196 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2832.0 | uss_MB: 1578.1 | child_rss_MB: 22919.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 24.3 +rank0 [2026-11-14 05:59:19,059] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:59:19,105] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:59:48,651] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:59:48,651] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:29,621] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 06:00:29,621] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:33,275] INFO : Step 45000/500000 rank0 | Train Loss: 7.7389 | Train Loss Unweighted: 3.7186 | Train Loss Components: Classification: 0.0291 | Classification_binary: 3.3569 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1547 | Regression_eta: 0.0077 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3231 | Classification_binary: 3.3569 | Regression_cos_phi: 0.0644 | Regression_energy: 1.6773 | Regression_eta: 0.0609 | Regression_pt: 2.1534 | Regression_sin_phi: 0.1029 | LR: 1.19e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7621s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2106 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2832.1 | uss_MB: 1578.2 | child_rss_MB: 22920.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:01:11,691] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 06:01:11,691] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:47,398] INFO : Step 46000/500000 rank0 | Train Loss: 7.6446 | Train Loss Unweighted: 3.7153 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.3588 | Regression_cos_phi: 0.0121 | Regression_energy: 0.1505 | Regression_eta: 0.0108 | Regression_pt: 0.1446 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3298 | Classification_binary: 3.3588 | Regression_cos_phi: 0.1338 | Regression_energy: 1.6312 | Regression_eta: 0.0852 | Regression_pt: 2.0105 | Regression_sin_phi: 0.0953 | LR: 1.23e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7584s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0262 | max_loss: 0.0025 | max_backward: 0.0261 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.2162 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2829.6 | uss_MB: 1575.7 | child_rss_MB: 22918.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:01:47,400] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 06:01:47,447] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 06:01:53,591] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 06:01:53,591] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:35,236] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 06:02:35,236] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:02,221] INFO : Step 47000/500000 rank0 | Train Loss: 7.6868 | Train Loss Unweighted: 3.5246 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.1522 | Regression_cos_phi: 0.0047 | Regression_energy: 0.1645 | Regression_eta: 0.0053 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3178 | Classification_binary: 3.1522 | Regression_cos_phi: 0.0522 | Regression_energy: 1.7830 | Regression_eta: 0.0422 | Regression_pt: 2.2889 | Regression_sin_phi: 0.0505 | LR: 1.27e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7753s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0024 | max_backward: 0.0272 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2182 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2831.6 | uss_MB: 1577.8 | child_rss_MB: 22918.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:03:17,626] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 06:03:17,626] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:59,910] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 06:03:59,910] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:17,755] INFO : Step 48000/500000 rank0 | Train Loss: 7.6529 | Train Loss Unweighted: 3.5439 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.1705 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1575 | Regression_eta: 0.0129 | Regression_pt: 0.1510 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3276 | Classification_binary: 3.1705 | Regression_cos_phi: 0.1169 | Regression_energy: 1.7073 | Regression_eta: 0.1017 | Regression_pt: 2.1004 | Regression_sin_phi: 0.1285 | LR: 1.31e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7664s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2169 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2831.7 | uss_MB: 1577.8 | child_rss_MB: 22919.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:04:17,756] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 06:04:17,804] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:04:42,402] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 06:04:42,403] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:24,042] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 06:05:24,042] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:32,594] INFO : Step 49000/500000 rank0 | Train Loss: 8.0698 | Train Loss Unweighted: 3.7802 | Train Loss Components: Classification: 0.0343 | Classification_binary: 3.3932 | Regression_cos_phi: 0.0159 | Regression_energy: 0.1550 | Regression_eta: 0.0077 | Regression_pt: 0.1607 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3809 | Classification_binary: 3.3932 | Regression_cos_phi: 0.1753 | Regression_energy: 1.6803 | Regression_eta: 0.0609 | Regression_pt: 2.2356 | Regression_sin_phi: 0.1436 | LR: 1.36e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7656s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2149 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2831.7 | uss_MB: 1577.9 | child_rss_MB: 22918.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:06,328] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 06:06:06,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:06:47,458] INFO : Step 50000/500000 rank0 | Train Loss: 7.9159 | Train Loss Unweighted: 3.7908 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.4212 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1631 | Regression_eta: 0.0050 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3063 | Classification_binary: 3.4212 | Regression_cos_phi: 0.0662 | Regression_energy: 1.7680 | Regression_eta: 0.0393 | Regression_pt: 2.2437 | Regression_sin_phi: 0.0713 | LR: 1.40e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7668s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0260 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2173 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2828.9 | uss_MB: 1575.0 | child_rss_MB: 22919.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:47,460] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 06:06:47,506] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:06:47,506] INFO : Running validation on rank0 +rank0 [2026-11-14 06:06:47,506] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:06:47,506] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:06:47,629] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 06:06:47,712] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:06:47,792] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:06:47,842] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:06:47,889] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:06:47,935] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:06:47,982] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:06:48,029] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:06:48,097] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:06:48,175] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:06:48,253] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:06:48,300] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:06:48,346] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:06:48,393] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:06:48,440] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:06:48,491] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:06:48,562] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:06:48,567] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:06:48,569] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:06:48,570] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:06:48,572] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:06:48,574] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:06:48,575] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:06:48,577] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:06:48,579] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:06:48,580] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:06:48,582] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:06:48,583] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:06:48,585] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:06:48,587] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:06:48,588] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:06:48,590] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:06:49,580] INFO : VALIDATION | Step=50000/500000 | Train Loss=7.9159 | Valid Loss=8.7096 | Train Loss Unweighted=3.7908 | Valid Loss Unweighted=3.9263 | Stale=0 | ETA=563.9m +rank0 [2026-11-14 06:06:49,581] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:06:49,698] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:06:49,707] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:06:49,773] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:06:50,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:06:50,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:06:50,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:06:50,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:06:50,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:06:51,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:06:51,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:06:51,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:06:51,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:06:51,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:06:51,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:06:52,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:06:52,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:06:52,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:06:52,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:06:52,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:06:52,817] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:06:52,824] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:06:53,207] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:06:53,446] INFO : Plotted number of elements +rank0 [2026-11-14 06:06:54,380] INFO : Plotted jets +rank0 [2026-11-14 06:06:55,081] INFO : Plotted jet ratio +rank0 [2026-11-14 06:06:56,836] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 06:06:56,836] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:38,647] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 06:07:38,648] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:08:10,683] INFO : Step 51000/500000 rank0 | Train Loss: 6.2996 | Train Loss Unweighted: 3.2629 | Train Loss Components: Classification: 0.0270 | Classification_binary: 2.9900 | Regression_cos_phi: 0.0034 | Regression_energy: 0.1087 | Regression_eta: 0.0059 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2993 | Classification_binary: 2.9900 | Regression_cos_phi: 0.0377 | Regression_energy: 1.1788 | Regression_eta: 0.0469 | Regression_pt: 1.6391 | Regression_sin_phi: 0.1078 | LR: 1.44e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7786s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0025 | max_backward: 0.0273 | max_optimizer_step: 0.0870 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2962.6 | uss_MB: 1708.7 | child_rss_MB: 22919.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:08:21,235] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 06:08:21,235] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:03,118] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 06:09:03,118] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:25,793] INFO : Step 52000/500000 rank0 | Train Loss: 7.4622 | Train Loss Unweighted: 3.3252 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.9541 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1613 | Regression_eta: 0.0080 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2500 | Classification_binary: 2.9541 | Regression_cos_phi: 0.1181 | Regression_energy: 1.7487 | Regression_eta: 0.0636 | Regression_pt: 2.2686 | Regression_sin_phi: 0.0592 | LR: 1.49e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7676s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0240 | max_loss: 0.0026 | max_backward: 0.0273 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2127 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2962.8 | uss_MB: 1709.0 | child_rss_MB: 22918.2 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:09:25,795] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 06:09:25,842] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 06:09:45,590] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 06:09:45,590] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:10:27,454] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 06:10:27,454] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:10:40,904] INFO : Step 53000/500000 rank0 | Train Loss: 7.8325 | Train Loss Unweighted: 3.6036 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.2239 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1624 | Regression_eta: 0.0066 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3267 | Classification_binary: 3.2239 | Regression_cos_phi: 0.1215 | Regression_energy: 1.7604 | Regression_eta: 0.0519 | Regression_pt: 2.2773 | Regression_sin_phi: 0.0709 | LR: 1.53e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7752s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0269 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.2172 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2962.6 | uss_MB: 1708.8 | child_rss_MB: 22920.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:11:09,811] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 06:11:09,811] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:53,513] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 06:11:53,515] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:57,821] INFO : Step 54000/500000 rank0 | Train Loss: 7.4899 | Train Loss Unweighted: 3.4385 | Train Loss Components: Classification: 0.0243 | Classification_binary: 3.0723 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1550 | Regression_eta: 0.0116 | Regression_pt: 0.1546 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2694 | Classification_binary: 3.0723 | Regression_cos_phi: 0.1315 | Regression_energy: 1.6804 | Regression_eta: 0.0917 | Regression_pt: 2.1502 | Regression_sin_phi: 0.0944 | LR: 1.58e-04 | DataLoad Time: 0.0163s | Model Forward Time: 0.7880s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0250 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0896 | max_tensorboard_logging: 0.2396 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2856.2 | uss_MB: 1602.4 | child_rss_MB: 22919.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:11:57,823] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 06:11:57,904] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 06:12:36,129] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 06:12:36,129] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:13,323] INFO : Step 55000/500000 rank0 | Train Loss: 7.1302 | Train Loss Unweighted: 3.6363 | Train Loss Components: Classification: 0.0361 | Classification_binary: 3.3210 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1284 | Regression_eta: 0.0077 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4010 | Classification_binary: 3.3210 | Regression_cos_phi: 0.0546 | Regression_energy: 1.3917 | Regression_eta: 0.0607 | Regression_pt: 1.8312 | Regression_sin_phi: 0.0700 | LR: 1.62e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7705s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0270 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.2203 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2859.6 | uss_MB: 1605.8 | child_rss_MB: 22921.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:13:18,950] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 06:13:18,950] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:14:00,798] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 06:14:00,798] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:14:28,421] INFO : Step 56000/500000 rank0 | Train Loss: 5.5289 | Train Loss Unweighted: 3.0077 | Train Loss Components: Classification: 0.0254 | Classification_binary: 2.7794 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0946 | Regression_eta: 0.0059 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2819 | Classification_binary: 2.7794 | Regression_cos_phi: 0.0427 | Regression_energy: 1.0254 | Regression_eta: 0.0465 | Regression_pt: 1.2893 | Regression_sin_phi: 0.0637 | LR: 1.67e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7813s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0865 | max_tensorboard_logging: 0.2348 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2860.1 | uss_MB: 1606.3 | child_rss_MB: 22922.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:14:28,422] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 06:14:28,472] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 06:14:43,398] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 06:14:43,398] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:25,239] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 06:15:25,239] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:43,665] INFO : Step 57000/500000 rank0 | Train Loss: 6.6769 | Train Loss Unweighted: 3.4412 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.1476 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1157 | Regression_eta: 0.0123 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3018 | Classification_binary: 3.1476 | Regression_cos_phi: 0.0974 | Regression_energy: 1.2541 | Regression_eta: 0.0976 | Regression_pt: 1.7002 | Regression_sin_phi: 0.0782 | LR: 1.72e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7801s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0314 | max_optimizer_step: 0.0871 | max_tensorboard_logging: 0.2198 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2861.2 | uss_MB: 1607.3 | child_rss_MB: 22922.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:16:07,774] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 06:16:07,775] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:16:49,940] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 06:16:49,940] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:16:59,097] INFO : Step 58000/500000 rank0 | Train Loss: 6.6022 | Train Loss Unweighted: 3.6380 | Train Loss Components: Classification: 0.0354 | Classification_binary: 3.3687 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1049 | Regression_eta: 0.0061 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3928 | Classification_binary: 3.3687 | Regression_cos_phi: 0.1116 | Regression_energy: 1.1369 | Regression_eta: 0.0481 | Regression_pt: 1.4577 | Regression_sin_phi: 0.0863 | LR: 1.76e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.8059s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0024 | max_backward: 0.0282 | max_optimizer_step: 0.0878 | max_tensorboard_logging: 0.2202 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.5 | uss_MB: 1601.7 | child_rss_MB: 22920.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:16:59,098] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 06:16:59,149] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 06:17:32,372] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 06:17:32,372] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:12,971] INFO : Step 59000/500000 rank0 | Train Loss: 7.0903 | Train Loss Unweighted: 3.4728 | Train Loss Components: Classification: 0.0349 | Classification_binary: 3.1450 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1325 | Regression_eta: 0.0099 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3872 | Classification_binary: 3.1450 | Regression_cos_phi: 0.0936 | Regression_energy: 1.4368 | Regression_eta: 0.0783 | Regression_pt: 1.8620 | Regression_sin_phi: 0.0874 | LR: 1.81e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7766s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0263 | max_optimizer_step: 0.0871 | max_tensorboard_logging: 0.2206 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2860.1 | uss_MB: 1606.3 | child_rss_MB: 22921.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:18:13,694] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 06:18:13,694] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:54,369] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 06:18:54,369] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:19:27,194] INFO : Step 60000/500000 rank0 | Train Loss: 6.6738 | Train Loss Unweighted: 3.4087 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.1107 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1233 | Regression_eta: 0.0118 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3017 | Classification_binary: 3.1107 | Regression_cos_phi: 0.1085 | Regression_energy: 1.3365 | Regression_eta: 0.0933 | Regression_pt: 1.6258 | Regression_sin_phi: 0.0974 | LR: 1.86e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7709s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2198 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2860.2 | uss_MB: 1606.4 | child_rss_MB: 22914.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:19:27,196] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 06:19:27,248] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 06:19:27,249] INFO : Running validation on rank0 +rank0 [2026-11-14 06:19:27,249] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:19:27,249] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:19:27,355] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 06:19:27,437] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:19:27,536] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:19:27,587] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:19:27,633] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:19:27,679] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:19:27,726] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:19:27,772] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:19:27,837] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:19:27,915] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:19:27,992] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:19:28,039] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:19:28,084] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:19:28,130] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:19:28,176] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:19:28,222] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:19:28,291] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:19:28,296] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:19:28,297] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:19:28,299] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:19:28,300] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:19:28,302] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:19:28,304] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:19:28,306] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:19:28,307] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:19:28,310] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:19:28,311] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:19:28,312] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:19:28,313] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:19:28,315] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:19:28,316] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:19:28,318] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:19:29,292] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.6738 | Valid Loss=8.9453 | Train Loss Unweighted=3.4087 | Valid Loss Unweighted=3.9658 | Stale=1 | ETA=552.3m +rank0 [2026-11-14 06:19:29,292] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:19:29,395] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:19:29,403] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:19:29,454] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:19:29,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:19:30,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:19:30,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:19:30,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:19:30,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:19:30,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:19:30,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:19:31,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:19:31,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:19:31,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:19:31,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:19:31,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:19:31,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:19:32,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:19:32,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:19:32,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:19:32,405] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:19:32,429] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:19:32,816] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:19:33,063] INFO : Plotted number of elements +rank0 [2026-11-14 06:19:33,989] INFO : Plotted jets +rank0 [2026-11-14 06:19:34,678] INFO : Plotted jet ratio +rank0 [2026-11-14 06:19:45,038] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 06:19:45,038] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:26,490] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 06:20:26,490] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:49,097] INFO : Step 61000/500000 rank0 | Train Loss: 6.0582 | Train Loss Unweighted: 3.0949 | Train Loss Components: Classification: 0.0252 | Classification_binary: 2.8267 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1091 | Regression_eta: 0.0067 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2793 | Classification_binary: 2.8267 | Regression_cos_phi: 0.0886 | Regression_energy: 1.1826 | Regression_eta: 0.0531 | Regression_pt: 1.5253 | Regression_sin_phi: 0.1024 | LR: 1.91e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7888s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0326 | max_optimizer_step: 0.0874 | max_tensorboard_logging: 0.2170 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2991.5 | uss_MB: 1737.6 | child_rss_MB: 22912.5 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:21:08,228] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 06:21:08,228] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:21:49,945] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 06:21:49,945] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:04,037] INFO : Step 62000/500000 rank0 | Train Loss: 6.4799 | Train Loss Unweighted: 3.3760 | Train Loss Components: Classification: 0.0217 | Classification_binary: 3.0985 | Regression_cos_phi: 0.0032 | Regression_energy: 0.1223 | Regression_eta: 0.0037 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2411 | Classification_binary: 3.0985 | Regression_cos_phi: 0.0353 | Regression_energy: 1.3264 | Regression_eta: 0.0292 | Regression_pt: 1.7117 | Regression_sin_phi: 0.0378 | LR: 1.95e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.8108s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.2181 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2883.7 | uss_MB: 1629.9 | child_rss_MB: 22910.8 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:22:04,039] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 06:22:04,091] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 06:22:32,403] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 06:22:32,403] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:13,548] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 06:23:13,548] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:18,291] INFO : Step 63000/500000 rank0 | Train Loss: 6.5941 | Train Loss Unweighted: 3.2361 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.9347 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1298 | Regression_eta: 0.0046 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2471 | Classification_binary: 2.9347 | Regression_cos_phi: 0.0698 | Regression_energy: 1.4076 | Regression_eta: 0.0367 | Regression_pt: 1.8116 | Regression_sin_phi: 0.0866 | LR: 2.00e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7743s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0024 | max_backward: 0.0259 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.2196 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2887.5 | uss_MB: 1633.7 | child_rss_MB: 22913.9 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:23:55,706] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 06:23:55,706] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:33,061] INFO : Step 64000/500000 rank0 | Train Loss: 5.9262 | Train Loss Unweighted: 3.2268 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.9823 | Regression_cos_phi: 0.0070 | Regression_energy: 0.0991 | Regression_eta: 0.0076 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2557 | Classification_binary: 2.9823 | Regression_cos_phi: 0.0769 | Regression_energy: 1.0745 | Regression_eta: 0.0601 | Regression_pt: 1.4012 | Regression_sin_phi: 0.0755 | LR: 2.05e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7738s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0025 | max_backward: 0.0272 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2130 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2887.6 | uss_MB: 1633.8 | child_rss_MB: 22911.9 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:24:33,062] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 06:24:33,110] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 06:24:38,156] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 06:24:38,157] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:19,994] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 06:25:19,994] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:48,555] INFO : Step 65000/500000 rank0 | Train Loss: 5.4422 | Train Loss Unweighted: 3.0169 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.7971 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0873 | Regression_eta: 0.0085 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2284 | Classification_binary: 2.7971 | Regression_cos_phi: 0.0518 | Regression_energy: 0.9466 | Regression_eta: 0.0673 | Regression_pt: 1.2775 | Regression_sin_phi: 0.0736 | LR: 2.10e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7666s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0024 | max_backward: 0.0273 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2135 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2887.8 | uss_MB: 1633.9 | child_rss_MB: 22912.7 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:26:02,767] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 06:26:02,767] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:44,512] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 06:26:44,512] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:27:03,494] INFO : Step 66000/500000 rank0 | Train Loss: 5.6695 | Train Loss Unweighted: 2.9567 | Train Loss Components: Classification: 0.0256 | Classification_binary: 2.7120 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1014 | Regression_eta: 0.0051 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2836 | Classification_binary: 2.7120 | Regression_cos_phi: 0.0582 | Regression_energy: 1.0991 | Regression_eta: 0.0401 | Regression_pt: 1.4159 | Regression_sin_phi: 0.0606 | LR: 2.15e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7604s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0024 | max_backward: 0.0272 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2165 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2883.1 | uss_MB: 1629.3 | child_rss_MB: 22912.6 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:27:03,496] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 06:27:03,548] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 06:27:27,332] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 06:27:27,332] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:08,975] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 06:28:08,975] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:18,789] INFO : Step 67000/500000 rank0 | Train Loss: 5.3649 | Train Loss Unweighted: 2.9360 | Train Loss Components: Classification: 0.0234 | Classification_binary: 2.7167 | Regression_cos_phi: 0.0051 | Regression_energy: 0.0903 | Regression_eta: 0.0052 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2598 | Classification_binary: 2.7167 | Regression_cos_phi: 0.0564 | Regression_energy: 0.9788 | Regression_eta: 0.0411 | Regression_pt: 1.2636 | Regression_sin_phi: 0.0486 | LR: 2.20e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.8091s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0879 | max_tensorboard_logging: 0.2122 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2886.5 | uss_MB: 1632.7 | child_rss_MB: 22913.4 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:28:51,860] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 06:28:51,860] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:29:34,047] INFO : Step 68000/500000 rank0 | Train Loss: 5.3894 | Train Loss Unweighted: 3.0968 | Train Loss Components: Classification: 0.0304 | Classification_binary: 2.8877 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0839 | Regression_eta: 0.0064 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3379 | Classification_binary: 2.8877 | Regression_cos_phi: 0.0467 | Regression_energy: 0.9095 | Regression_eta: 0.0504 | Regression_pt: 1.1088 | Regression_sin_phi: 0.0485 | LR: 2.25e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7599s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2075 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2886.6 | uss_MB: 1632.8 | child_rss_MB: 22913.8 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 16.3 +rank0 [2026-11-14 06:29:34,048] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 06:29:34,095] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 06:29:34,240] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 06:29:34,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:15,959] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 06:30:15,959] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:48,987] INFO : Step 69000/500000 rank0 | Train Loss: 5.6626 | Train Loss Unweighted: 3.0507 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.8122 | Regression_cos_phi: 0.0088 | Regression_energy: 0.0964 | Regression_eta: 0.0093 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2513 | Classification_binary: 2.8122 | Regression_cos_phi: 0.0969 | Regression_energy: 1.0447 | Regression_eta: 0.0732 | Regression_pt: 1.2913 | Regression_sin_phi: 0.0930 | LR: 2.30e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7634s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0268 | max_optimizer_step: 0.0858 | max_tensorboard_logging: 0.2151 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2886.8 | uss_MB: 1633.0 | child_rss_MB: 22914.3 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:30:58,369] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 06:30:58,369] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:31:40,059] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 06:31:40,059] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:04,282] INFO : Step 70000/500000 rank0 | Train Loss: 6.0207 | Train Loss Unweighted: 3.1402 | Train Loss Components: Classification: 0.0256 | Classification_binary: 2.8736 | Regression_cos_phi: 0.0145 | Regression_energy: 0.1053 | Regression_eta: 0.0130 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0145 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2841 | Classification_binary: 2.8736 | Regression_cos_phi: 0.1599 | Regression_energy: 1.1421 | Regression_eta: 0.1031 | Regression_pt: 1.3016 | Regression_sin_phi: 0.1563 | LR: 2.35e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7762s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2237 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2883.2 | uss_MB: 1629.4 | child_rss_MB: 22915.4 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:32:04,284] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 06:32:04,331] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 06:32:04,332] INFO : Running validation on rank0 +rank0 [2026-11-14 06:32:04,332] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:32:04,332] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:32:04,438] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 06:32:04,523] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:32:04,602] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:32:04,653] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:32:04,699] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:32:04,745] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:32:04,791] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:32:04,839] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:32:04,904] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:32:04,982] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:32:05,060] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:32:05,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:32:05,153] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:32:05,198] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:32:05,243] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:32:05,290] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:32:05,361] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:32:05,366] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:32:05,368] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:32:05,369] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:32:05,371] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:32:05,372] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:32:05,374] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:32:05,376] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:32:05,377] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:32:05,379] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:32:05,380] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:32:05,382] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:32:05,383] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:32:05,385] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:32:05,386] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:32:05,388] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:32:06,373] INFO : VALIDATION | Step=70000/500000 | Train Loss=6.0207 | Valid Loss=9.5168 | Train Loss Unweighted=3.1402 | Valid Loss Unweighted=4.1249 | Stale=2 | ETA=540.2m +rank0 [2026-11-14 06:32:06,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:32:06,474] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:32:06,482] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:32:06,534] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:32:06,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:32:07,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:32:07,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:32:07,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:32:07,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:32:07,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:32:07,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:32:08,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:32:08,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:32:08,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:32:08,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:32:08,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:32:08,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:32:09,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:32:09,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:32:09,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:32:09,486] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:32:09,493] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:32:09,874] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:32:10,126] INFO : Plotted number of elements +rank0 [2026-11-14 06:32:11,060] INFO : Plotted jets +rank0 [2026-11-14 06:32:11,751] INFO : Plotted jet ratio +rank0 [2026-11-14 06:32:30,808] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 06:32:30,808] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:33:12,796] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 06:33:12,796] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:33:27,518] INFO : Step 71000/500000 rank0 | Train Loss: 5.1183 | Train Loss Unweighted: 2.8594 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.6540 | Regression_cos_phi: 0.0057 | Regression_energy: 0.0851 | Regression_eta: 0.0077 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.1929 | Classification_binary: 2.6540 | Regression_cos_phi: 0.0633 | Regression_energy: 0.9231 | Regression_eta: 0.0611 | Regression_pt: 1.1565 | Regression_sin_phi: 0.0674 | LR: 2.40e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7760s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0270 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2214 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3011.4 | uss_MB: 1757.7 | child_rss_MB: 22917.3 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:33:55,386] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 06:33:55,387] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:37,168] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 06:34:37,169] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:42,653] INFO : Step 72000/500000 rank0 | Train Loss: 5.0601 | Train Loss Unweighted: 2.8020 | Train Loss Components: Classification: 0.0268 | Classification_binary: 2.5958 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0822 | Regression_eta: 0.0080 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2972 | Classification_binary: 2.5958 | Regression_cos_phi: 0.0529 | Regression_energy: 0.8908 | Regression_eta: 0.0635 | Regression_pt: 1.1111 | Regression_sin_phi: 0.0487 | LR: 2.45e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7813s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0024 | max_backward: 0.0274 | max_optimizer_step: 0.0879 | max_tensorboard_logging: 0.2178 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3011.5 | uss_MB: 1757.7 | child_rss_MB: 22917.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 06:34:42,655] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 06:34:42,701] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 06:35:19,919] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 06:35:19,919] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:57,781] INFO : Step 73000/500000 rank0 | Train Loss: 5.5330 | Train Loss Unweighted: 2.9531 | Train Loss Components: Classification: 0.0271 | Classification_binary: 2.7190 | Regression_cos_phi: 0.0099 | Regression_energy: 0.0915 | Regression_eta: 0.0051 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3012 | Classification_binary: 2.7190 | Regression_cos_phi: 0.1092 | Regression_energy: 0.9924 | Regression_eta: 0.0407 | Regression_pt: 1.2831 | Regression_sin_phi: 0.0874 | LR: 2.50e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.8127s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0275 | max_optimizer_step: 0.0868 | max_tensorboard_logging: 0.2149 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3011.5 | uss_MB: 1757.7 | child_rss_MB: 22918.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:36:02,224] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 06:36:02,224] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:43,914] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 06:36:43,914] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:12,579] INFO : Step 74000/500000 rank0 | Train Loss: 5.3633 | Train Loss Unweighted: 3.0301 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.8185 | Regression_cos_phi: 0.0065 | Regression_energy: 0.0860 | Regression_eta: 0.0068 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2156 | Classification_binary: 2.8185 | Regression_cos_phi: 0.0720 | Regression_energy: 0.9324 | Regression_eta: 0.0539 | Regression_pt: 1.1978 | Regression_sin_phi: 0.0732 | LR: 2.55e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7869s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0036 | max_backward: 0.0274 | max_optimizer_step: 0.0888 | max_tensorboard_logging: 0.2222 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2907.9 | uss_MB: 1654.2 | child_rss_MB: 22917.4 | allocated_MB: 126.1 | reserved_MB: 628.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 06:37:12,580] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 06:37:12,629] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 06:37:26,298] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 06:37:26,298] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:07,631] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 06:38:07,631] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:27,513] INFO : Step 75000/500000 rank0 | Train Loss: 4.9342 | Train Loss Unweighted: 2.7552 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5549 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0809 | Regression_eta: 0.0099 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2266 | Classification_binary: 2.5549 | Regression_cos_phi: 0.0764 | Regression_energy: 0.8767 | Regression_eta: 0.0784 | Regression_pt: 1.0493 | Regression_sin_phi: 0.0718 | LR: 2.60e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7732s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0262 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2205 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2911.2 | uss_MB: 1657.5 | child_rss_MB: 22918.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:38:49,798] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 06:38:49,798] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:39:30,070] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 06:39:30,070] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:39:40,358] INFO : Step 76000/500000 rank0 | Train Loss: 4.7154 | Train Loss Unweighted: 2.6986 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.5163 | Regression_cos_phi: 0.0082 | Regression_energy: 0.0709 | Regression_eta: 0.0051 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.1805 | Classification_binary: 2.5163 | Regression_cos_phi: 0.0907 | Regression_energy: 0.7690 | Regression_eta: 0.0406 | Regression_pt: 1.0528 | Regression_sin_phi: 0.0655 | LR: 2.65e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7697s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2176 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2911.3 | uss_MB: 1657.6 | child_rss_MB: 22918.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:39:40,360] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 06:39:40,407] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 06:40:12,484] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 06:40:12,484] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:53,966] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 06:40:53,966] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:55,140] INFO : Step 77000/500000 rank0 | Train Loss: 4.9293 | Train Loss Unweighted: 2.9131 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7289 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0690 | Regression_eta: 0.0059 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2293 | Classification_binary: 2.7289 | Regression_cos_phi: 0.0663 | Regression_energy: 0.7481 | Regression_eta: 0.0465 | Regression_pt: 0.9784 | Regression_sin_phi: 0.1319 | LR: 2.70e-04 | DataLoad Time: 0.0003s | Model Forward Time: 0.7683s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0271 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0857 | max_tensorboard_logging: 0.2141 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2911.5 | uss_MB: 1657.8 | child_rss_MB: 22917.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:41:36,398] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 06:41:36,398] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:09,714] INFO : Step 78000/500000 rank0 | Train Loss: 4.6972 | Train Loss Unweighted: 2.7253 | Train Loss Components: Classification: 0.0258 | Classification_binary: 2.5448 | Regression_cos_phi: 0.0064 | Regression_energy: 0.0681 | Regression_eta: 0.0060 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2864 | Classification_binary: 2.5448 | Regression_cos_phi: 0.0710 | Regression_energy: 0.7379 | Regression_eta: 0.0471 | Regression_pt: 0.9356 | Regression_sin_phi: 0.0745 | LR: 2.75e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7768s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0252 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2172 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2911.6 | uss_MB: 1657.8 | child_rss_MB: 22918.5 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:42:09,715] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 06:42:09,774] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 06:42:18,359] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 06:42:18,359] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:58,746] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 06:42:58,747] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:43:22,766] INFO : Step 79000/500000 rank0 | Train Loss: 4.5664 | Train Loss Unweighted: 2.5385 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.3515 | Regression_cos_phi: 0.0096 | Regression_energy: 0.0735 | Regression_eta: 0.0075 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2415 | Classification_binary: 2.3515 | Regression_cos_phi: 0.1066 | Regression_energy: 0.7973 | Regression_eta: 0.0592 | Regression_pt: 0.9200 | Regression_sin_phi: 0.0902 | LR: 2.80e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7643s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2167 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2911.3 | uss_MB: 1722.3 | child_rss_MB: 22918.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:43:40,721] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 06:43:40,721] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:44:22,378] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 06:44:22,379] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:44:38,014] INFO : Step 80000/500000 rank0 | Train Loss: 4.5300 | Train Loss Unweighted: 2.5459 | Train Loss Components: Classification: 0.0241 | Classification_binary: 2.3645 | Regression_cos_phi: 0.0071 | Regression_energy: 0.0683 | Regression_eta: 0.0050 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2671 | Classification_binary: 2.3645 | Regression_cos_phi: 0.0785 | Regression_energy: 0.7403 | Regression_eta: 0.0398 | Regression_pt: 0.9385 | Regression_sin_phi: 0.1014 | LR: 2.85e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7569s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0251 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.2145 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2912.0 | uss_MB: 1723.0 | child_rss_MB: 22918.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:44:38,015] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 06:44:38,062] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 06:44:38,063] INFO : Running validation on rank0 +rank0 [2026-11-14 06:44:38,063] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:44:38,063] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:44:38,167] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 06:44:38,251] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:44:38,329] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:44:38,379] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:44:38,425] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:44:38,470] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:44:38,517] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:44:38,563] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:44:38,632] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:44:38,710] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:44:38,788] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:44:38,836] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:44:38,883] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:44:38,929] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:44:38,975] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:44:39,022] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:44:39,096] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:44:39,101] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:44:39,102] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:44:39,104] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:44:39,105] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:44:39,107] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:44:39,109] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:44:39,110] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:44:39,112] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:44:39,113] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:44:39,115] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:44:39,116] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:44:39,118] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:44:39,120] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:44:39,121] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:44:39,123] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:44:40,066] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.5300 | Valid Loss=10.2606 | Train Loss Unweighted=2.5459 | Valid Loss Unweighted=4.5677 | Stale=3 | ETA=527.6m +rank0 [2026-11-14 06:44:40,066] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:44:40,167] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:44:40,176] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:40,228] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:44:40,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:44:40,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:44:41,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:44:41,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:44:41,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:44:41,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:44:41,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:44:41,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:44:41,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:44:42,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:44:42,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:44:42,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:44:42,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:44:42,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:44:42,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:44:43,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:44:43,204] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:44:43,212] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:43,600] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:43,838] INFO : Plotted number of elements +rank0 [2026-11-14 06:44:44,791] INFO : Plotted jets +rank0 [2026-11-14 06:44:45,497] INFO : Plotted jet ratio +rank0 [2026-11-14 06:44:45,972] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 06:44:45,972] INFO : Training completed. Total time on device 0: 100.594min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..0186755156f95018e87996615b0f84522754df7c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/train.log.0 @@ -0,0 +1,1351 @@ +rank0 [2026-11-14 05:04:09,351] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 05:04:09,351] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 05:04:09,351] INFO : Instantiating model +rank0 [2026-11-14 05:04:09,369] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 05:04:09,370] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 05:04:09,371] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 05:04:09,371] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 05:04:09,371] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 05:04:09,371] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 05:04:09,371] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,373] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 05:04:09,373] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,375] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 05:04:09,375] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,378] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 05:04:09,378] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,380] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 05:04:09,380] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,383] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 05:04:09,383] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,385] INFO : Backbone initialization took 0.01s +rank0 [2026-11-14 05:04:09,385] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 05:04:09,386] INFO : Initializing output DNNs +rank0 [2026-11-14 05:04:09,386] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,389] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:04:09,395] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 05:04:09,395] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 05:04:09,395] INFO : jagged_attention enabled=True +rank0 [2026-11-14 05:04:09,395] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 05:04:09,396] INFO : task_queries enabled=True +rank0 [2026-11-14 05:04:09,396] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 05:04:09,396] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 05:04:09,396] INFO : classification_query parameters: 256 +rank0 [2026-11-14 05:04:09,396] INFO : regression_query parameters: 256 +rank0 [2026-11-14 05:04:09,396] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 05:04:09,396] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 05:04:09,396] INFO : output_mode=elementwise +rank0 [2026-11-14 05:04:09,396] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 05:04:09,396] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 05:04:09,396] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 05:04:09,396] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 05:04:09,396] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 05:04:09,396] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 05:04:09,397] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 05:04:09,397] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 05:04:09,397] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 05:04:09,397] INFO : MLPF __init__ done +rank0 [2026-11-14 05:04:09,397] INFO : Instantiated model +rank0 [2026-11-14 05:04:09,398] INFO : Moving model to device rank=0 +rank0 [2026-11-14 05:04:09,402] INFO : Moved model to device rank=0 +rank0 [2026-11-14 05:04:09,403] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 05:04:09,462] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 05:04:09,462] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 05:04:09,463] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 05:04:09,463] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 05:04:09,464] INFO : Total parameters: 3910930 +rank0 [2026-11-14 05:04:09,465] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 05:04:09,466] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441 +rank0 [2026-11-14 05:04:09,466] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441 +rank0 [2026-11-14 05:04:09,466] INFO : Getting dataloaders +rank0 [2026-11-14 05:04:09,467] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:04:09,617] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,657] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,697] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,737] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,775] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,812] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,860] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,895] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:09,979] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:10,016] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:04:10,016] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 05:04:10,017] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 05:04:10,017] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 05:04:10,017] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:04:10,027] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,036] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,045] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,054] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,063] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,072] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,081] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,091] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,099] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,109] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:04:10,109] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 05:04:10,109] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 05:04:10,142] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:04:10,316] INFO : Got dataloaders +rank0 [2026-11-14 05:04:10,316] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 05:04:10,316] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 05:04:10,316] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 05:04:10,319] INFO : Creating train_iterator +rank0 [2026-11-14 05:04:10,320] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:04:10,350] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:04:10,520] INFO : Created train_iterator +rank0 [2026-11-14 05:04:19,288] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.148003125190735, 'Classification': 0.6440470790863038, 'Regression_pt': 0.5139240911602974, 'Regression_eta': 0.9037979310750961, 'Regression_sin_phi': 0.6648890712857246, 'Regression_cos_phi': 0.6466032195091248, 'Regression_energy': 0.6593179452419281}, weights={'Classification_binary': 1.0, 'Classification': 11.098572731018066, 'Regression_pt': 13.908675193786621, 'Regression_eta': 7.908851146697998, 'Regression_sin_phi': 10.750670433044434, 'Regression_cos_phi': 11.05469799041748, 'Regression_energy': 10.841511726379395} +rank0 [2026-11-14 05:04:54,142] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 05:04:54,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:05:27,283] INFO : Step 1000/500000 rank0 | Train Loss: 12.5896 | Train Loss Unweighted: 5.2980 | Train Loss Components: Classification: 0.0681 | Classification_binary: 4.6273 | Regression_cos_phi: 0.0269 | Regression_energy: 0.2531 | Regression_eta: 0.0397 | Regression_pt: 0.2563 | Regression_sin_phi: 0.0268 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.7557 | Classification_binary: 4.6273 | Regression_cos_phi: 0.2973 | Regression_energy: 2.7436 | Regression_eta: 0.3138 | Regression_pt: 3.5643 | Regression_sin_phi: 0.2878 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7491s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0237 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0833 | max_tensorboard_logging: 0.2184 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2630.3 | uss_MB: 1367.6 | child_rss_MB: 22909.3 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:05:36,220] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 05:05:36,221] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:16,554] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 05:06:16,555] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:06:39,851] INFO : Step 2000/500000 rank0 | Train Loss: 12.3836 | Train Loss Unweighted: 5.1447 | Train Loss Components: Classification: 0.0440 | Classification_binary: 4.4854 | Regression_cos_phi: 0.0268 | Regression_energy: 0.2679 | Regression_eta: 0.0306 | Regression_pt: 0.2690 | Regression_sin_phi: 0.0209 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4883 | Classification_binary: 4.4854 | Regression_cos_phi: 0.2964 | Regression_energy: 2.9050 | Regression_eta: 0.2422 | Regression_pt: 3.7416 | Regression_sin_phi: 0.2247 | LR: 2.02e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7514s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0844 | max_tensorboard_logging: 0.2161 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2632.2 | uss_MB: 1369.6 | child_rss_MB: 22908.4 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:06:39,853] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:06:58,001] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 05:06:58,001] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:07:38,358] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 05:07:38,358] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:07:52,609] INFO : Step 3000/500000 rank0 | Train Loss: 10.7572 | Train Loss Unweighted: 4.6472 | Train Loss Components: Classification: 0.0446 | Classification_binary: 4.0980 | Regression_cos_phi: 0.0123 | Regression_energy: 0.2292 | Regression_eta: 0.0139 | Regression_pt: 0.2390 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4946 | Classification_binary: 4.0980 | Regression_cos_phi: 0.1361 | Regression_energy: 2.4846 | Regression_eta: 0.1102 | Regression_pt: 3.3236 | Regression_sin_phi: 0.1100 | LR: 2.05e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7446s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0258 | max_optimizer_step: 0.0828 | max_tensorboard_logging: 0.2144 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2634.7 | uss_MB: 1372.4 | child_rss_MB: 22909.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:08:19,836] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 05:08:19,836] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:00,268] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 05:09:00,268] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:09:05,658] INFO : Step 4000/500000 rank0 | Train Loss: 9.2690 | Train Loss Unweighted: 4.4229 | Train Loss Components: Classification: 0.0429 | Classification_binary: 3.9843 | Regression_cos_phi: 0.0165 | Regression_energy: 0.1680 | Regression_eta: 0.0153 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4756 | Classification_binary: 3.9843 | Regression_cos_phi: 0.1829 | Regression_energy: 1.8219 | Regression_eta: 0.1214 | Regression_pt: 2.5442 | Regression_sin_phi: 0.1388 | LR: 2.08e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7502s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0267 | max_optimizer_step: 0.0834 | max_tensorboard_logging: 0.2149 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2635.2 | uss_MB: 1372.9 | child_rss_MB: 22912.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:09:05,660] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:09:41,771] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 05:09:41,771] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:10:19,690] INFO : Step 5000/500000 rank0 | Train Loss: 10.6934 | Train Loss Unweighted: 4.8094 | Train Loss Components: Classification: 0.0409 | Classification_binary: 4.2799 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2238 | Regression_eta: 0.0135 | Regression_pt: 0.2285 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4538 | Classification_binary: 4.2799 | Regression_cos_phi: 0.1174 | Regression_energy: 2.4261 | Regression_eta: 0.1067 | Regression_pt: 3.1778 | Regression_sin_phi: 0.1318 | LR: 2.13e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7881s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0269 | max_optimizer_step: 0.0837 | max_tensorboard_logging: 0.2154 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2634.6 | uss_MB: 1372.3 | child_rss_MB: 22911.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:10:23,896] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 05:10:23,896] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:04,114] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 05:11:04,115] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:11:32,497] INFO : Step 6000/500000 rank0 | Train Loss: 8.7115 | Train Loss Unweighted: 4.2087 | Train Loss Components: Classification: 0.0417 | Classification_binary: 3.8025 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1685 | Regression_eta: 0.0085 | Regression_pt: 0.1689 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4628 | Classification_binary: 3.8025 | Regression_cos_phi: 0.1150 | Regression_energy: 1.8265 | Regression_eta: 0.0671 | Regression_pt: 2.3489 | Regression_sin_phi: 0.0887 | LR: 2.19e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7392s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.0837 | max_tensorboard_logging: 0.2130 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2634.6 | uss_MB: 1372.3 | child_rss_MB: 22909.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:11:32,499] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:11:45,871] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 05:11:45,872] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:27,063] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 05:12:27,064] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:12:46,439] INFO : Step 7000/500000 rank0 | Train Loss: 10.3344 | Train Loss Unweighted: 4.4695 | Train Loss Components: Classification: 0.0342 | Classification_binary: 3.9430 | Regression_cos_phi: 0.0137 | Regression_energy: 0.2263 | Regression_eta: 0.0121 | Regression_pt: 0.2310 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3792 | Classification_binary: 3.9430 | Regression_cos_phi: 0.1512 | Regression_energy: 2.4537 | Regression_eta: 0.0956 | Regression_pt: 3.2129 | Regression_sin_phi: 0.0989 | LR: 2.26e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7569s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0278 | max_optimizer_step: 0.0852 | max_tensorboard_logging: 0.2123 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2665.3 | uss_MB: 1403.0 | child_rss_MB: 22914.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:13:09,121] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 05:13:09,122] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:49,539] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 05:13:49,539] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:13:59,919] INFO : Step 8000/500000 rank0 | Train Loss: 8.6836 | Train Loss Unweighted: 4.4360 | Train Loss Components: Classification: 0.0417 | Classification_binary: 4.0501 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1536 | Regression_eta: 0.0141 | Regression_pt: 0.1562 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4628 | Classification_binary: 4.0501 | Regression_cos_phi: 0.1104 | Regression_energy: 1.6654 | Regression_eta: 0.1118 | Regression_pt: 2.1719 | Regression_sin_phi: 0.1112 | LR: 2.34e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7752s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0238 | max_loss: 0.0023 | max_backward: 0.0266 | max_optimizer_step: 0.1116 | max_tensorboard_logging: 0.2125 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2665.4 | uss_MB: 1403.1 | child_rss_MB: 22910.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:13:59,921] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:13:59,967] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:14:31,710] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 05:14:31,710] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:13,105] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 05:15:13,105] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:15:14,365] INFO : Step 9000/500000 rank0 | Train Loss: 8.4548 | Train Loss Unweighted: 4.1895 | Train Loss Components: Classification: 0.0327 | Classification_binary: 3.8016 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1543 | Regression_eta: 0.0184 | Regression_pt: 0.1606 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3633 | Classification_binary: 3.8016 | Regression_cos_phi: 0.1153 | Regression_energy: 1.6725 | Regression_eta: 0.1455 | Regression_pt: 2.2337 | Regression_sin_phi: 0.1228 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.7438s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0267 | max_optimizer_step: 0.0833 | max_tensorboard_logging: 0.2200 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2634.0 | uss_MB: 1371.8 | child_rss_MB: 22909.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:15:55,171] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 05:15:55,171] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:16:29,023] INFO : Step 10000/500000 rank0 | Train Loss: 8.9702 | Train Loss Unweighted: 4.4252 | Train Loss Components: Classification: 0.0385 | Classification_binary: 4.0157 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1678 | Regression_eta: 0.0083 | Regression_pt: 0.1725 | Regression_sin_phi: 0.0124 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4272 | Classification_binary: 4.0157 | Regression_cos_phi: 0.1103 | Regression_energy: 1.8197 | Regression_eta: 0.0656 | Regression_pt: 2.3988 | Regression_sin_phi: 0.1328 | LR: 2.52e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7887s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0833 | max_tensorboard_logging: 0.2527 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2634.5 | uss_MB: 1372.2 | child_rss_MB: 22910.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:16:29,025] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:16:29,072] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:16:29,073] INFO : Running validation on rank0 +rank0 [2026-11-14 05:16:29,073] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:16:29,073] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:16:29,199] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 05:16:29,314] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:16:29,371] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:16:29,411] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:16:29,450] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:16:29,483] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:16:29,522] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:16:29,563] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:16:29,599] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:16:29,654] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:16:29,708] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:16:29,746] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:16:29,786] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:16:29,819] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:16:29,859] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:16:29,897] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:16:29,935] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:16:29,938] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:16:29,939] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:16:29,941] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:16:29,943] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:16:29,944] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:16:29,946] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:16:29,947] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:16:29,949] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:16:29,951] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:16:29,952] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:16:29,954] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:16:29,955] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:16:29,957] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:16:29,958] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:16:29,960] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:16:30,981] INFO : VALIDATION | Step=10000/500000 | Train Loss=8.9702 | Valid Loss=10.1504 | Train Loss Unweighted=4.4252 | Valid Loss Unweighted=4.5026 | Stale=0 | ETA=604.9m +rank0 [2026-11-14 05:16:30,981] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:16:31,080] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:16:31,088] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:16:31,137] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:16:31,586] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:16:31,752] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:16:31,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:16:32,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:16:32,250] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:16:32,410] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:16:32,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:16:32,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:16:32,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:16:33,061] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:16:33,220] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:16:33,380] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:16:33,538] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:16:33,697] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:16:33,857] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:16:34,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:16:34,096] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:16:34,114] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:16:34,500] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:16:34,805] INFO : Plotted number of elements +rank0 [2026-11-14 05:16:35,757] INFO : Plotted jets +rank0 [2026-11-14 05:16:36,465] INFO : Plotted jet ratio +rank0 [2026-11-14 05:16:45,473] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 05:16:45,474] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:27,081] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 05:17:27,082] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:17:51,628] INFO : Step 11000/500000 rank0 | Train Loss: 10.4528 | Train Loss Unweighted: 4.5009 | Train Loss Components: Classification: 0.0314 | Classification_binary: 3.9681 | Regression_cos_phi: 0.0086 | Regression_energy: 0.2310 | Regression_eta: 0.0130 | Regression_pt: 0.2405 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3486 | Classification_binary: 3.9681 | Regression_cos_phi: 0.0948 | Regression_energy: 2.5039 | Regression_eta: 0.1026 | Regression_pt: 3.3453 | Regression_sin_phi: 0.0895 | LR: 2.63e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7669s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.2170 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2817.6 | uss_MB: 1563.5 | child_rss_MB: 22913.9 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 05:18:09,432] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 05:18:09,433] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:18:51,213] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 05:18:51,213] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:19:06,729] INFO : Step 12000/500000 rank0 | Train Loss: 10.4375 | Train Loss Unweighted: 4.3321 | Train Loss Components: Classification: 0.0360 | Classification_binary: 3.7863 | Regression_cos_phi: 0.0090 | Regression_energy: 0.2385 | Regression_eta: 0.0085 | Regression_pt: 0.2441 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3996 | Classification_binary: 3.7863 | Regression_cos_phi: 0.0997 | Regression_energy: 2.5855 | Regression_eta: 0.0670 | Regression_pt: 3.3952 | Regression_sin_phi: 0.1040 | LR: 2.75e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.8108s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2589 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2817.6 | uss_MB: 1563.5 | child_rss_MB: 22915.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:19:06,731] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:19:06,778] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:19:33,925] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 05:19:33,925] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:15,160] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 05:20:15,160] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:20:21,313] INFO : Step 13000/500000 rank0 | Train Loss: 9.2974 | Train Loss Unweighted: 4.3393 | Train Loss Components: Classification: 0.0370 | Classification_binary: 3.8907 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1865 | Regression_eta: 0.0155 | Regression_pt: 0.1883 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4102 | Classification_binary: 3.8907 | Regression_cos_phi: 0.1031 | Regression_energy: 2.0221 | Regression_eta: 0.1225 | Regression_pt: 2.6184 | Regression_sin_phi: 0.1303 | LR: 2.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7612s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0847 | max_tensorboard_logging: 0.2187 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2745.1 | uss_MB: 1491.0 | child_rss_MB: 22914.7 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:20:57,085] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:20:57,085] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:21:35,330] INFO : Step 14000/500000 rank0 | Train Loss: 9.2849 | Train Loss Unweighted: 4.1741 | Train Loss Components: Classification: 0.0400 | Classification_binary: 3.7166 | Regression_cos_phi: 0.0043 | Regression_energy: 0.2015 | Regression_eta: 0.0060 | Regression_pt: 0.2005 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4444 | Classification_binary: 3.7166 | Regression_cos_phi: 0.0470 | Regression_energy: 2.1846 | Regression_eta: 0.0476 | Regression_pt: 2.7882 | Regression_sin_phi: 0.0566 | LR: 3.02e-05 | DataLoad Time: 0.0024s | Model Forward Time: 0.7628s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0886 | max_tensorboard_logging: 0.2137 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2745.4 | uss_MB: 1491.3 | child_rss_MB: 22916.3 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:21:35,332] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:21:35,378] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:21:39,018] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 05:21:39,018] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:20,071] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 05:22:20,071] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:22:49,803] INFO : Step 15000/500000 rank0 | Train Loss: 8.9437 | Train Loss Unweighted: 4.3029 | Train Loss Components: Classification: 0.0412 | Classification_binary: 3.8822 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1724 | Regression_eta: 0.0153 | Regression_pt: 0.1742 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4577 | Classification_binary: 3.8822 | Regression_cos_phi: 0.0846 | Regression_energy: 1.8691 | Regression_eta: 0.1211 | Regression_pt: 2.4228 | Regression_sin_phi: 0.1061 | LR: 3.17e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7621s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0252 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2156 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2748.3 | uss_MB: 1494.2 | child_rss_MB: 22917.3 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:23:02,684] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 05:23:02,684] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:23:43,379] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 05:23:43,379] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:24:03,342] INFO : Step 16000/500000 rank0 | Train Loss: 9.6332 | Train Loss Unweighted: 4.4402 | Train Loss Components: Classification: 0.0353 | Classification_binary: 3.9738 | Regression_cos_phi: 0.0117 | Regression_energy: 0.1946 | Regression_eta: 0.0084 | Regression_pt: 0.2013 | Regression_sin_phi: 0.0151 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3919 | Classification_binary: 3.9738 | Regression_cos_phi: 0.1293 | Regression_energy: 2.1095 | Regression_eta: 0.0664 | Regression_pt: 2.8004 | Regression_sin_phi: 0.1619 | LR: 3.33e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7652s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0250 | max_loss: 0.0024 | max_backward: 0.0276 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.2134 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2748.8 | uss_MB: 1494.7 | child_rss_MB: 22916.6 | allocated_MB: 126.1 | reserved_MB: 632.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:24:03,343] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:24:03,389] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:24:25,363] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 05:24:25,363] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:06,794] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 05:25:06,794] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:25:17,735] INFO : Step 17000/500000 rank0 | Train Loss: 10.2397 | Train Loss Unweighted: 4.1611 | Train Loss Components: Classification: 0.0404 | Classification_binary: 3.6110 | Regression_cos_phi: 0.0173 | Regression_energy: 0.2304 | Regression_eta: 0.0179 | Regression_pt: 0.2298 | Regression_sin_phi: 0.0142 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4486 | Classification_binary: 3.6110 | Regression_cos_phi: 0.1909 | Regression_energy: 2.4977 | Regression_eta: 0.1419 | Regression_pt: 3.1968 | Regression_sin_phi: 0.1529 | LR: 3.51e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8078s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2744.5 | uss_MB: 1490.4 | child_rss_MB: 22914.9 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:25:49,409] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 05:25:49,409] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:31,269] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 05:26:31,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:26:33,151] INFO : Step 18000/500000 rank0 | Train Loss: 9.1121 | Train Loss Unweighted: 4.1689 | Train Loss Components: Classification: 0.0355 | Classification_binary: 3.7259 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1896 | Regression_eta: 0.0065 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3939 | Classification_binary: 3.7259 | Regression_cos_phi: 0.1095 | Regression_energy: 2.0560 | Regression_eta: 0.0516 | Regression_pt: 2.6813 | Regression_sin_phi: 0.0939 | LR: 3.69e-05 | DataLoad Time: 0.0004s | Model Forward Time: 0.7652s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0269 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.2173 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2745.5 | uss_MB: 1491.4 | child_rss_MB: 22915.4 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:26:33,153] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:26:33,201] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:27:13,839] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 05:27:13,840] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:27:48,253] INFO : Step 19000/500000 rank0 | Train Loss: 7.8044 | Train Loss Unweighted: 3.9318 | Train Loss Components: Classification: 0.0372 | Classification_binary: 3.5818 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1433 | Regression_eta: 0.0094 | Regression_pt: 0.1452 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4132 | Classification_binary: 3.5818 | Regression_cos_phi: 0.0826 | Regression_energy: 1.5540 | Regression_eta: 0.0747 | Regression_pt: 2.0191 | Regression_sin_phi: 0.0790 | LR: 3.88e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7555s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0273 | max_optimizer_step: 0.0851 | max_tensorboard_logging: 0.2154 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2748.4 | uss_MB: 1494.3 | child_rss_MB: 22914.4 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:27:56,254] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 05:27:56,255] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:28:37,545] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 05:28:37,545] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:29:02,927] INFO : Step 20000/500000 rank0 | Train Loss: 10.1618 | Train Loss Unweighted: 4.3767 | Train Loss Components: Classification: 0.0400 | Classification_binary: 3.8541 | Regression_cos_phi: 0.0090 | Regression_energy: 0.2319 | Regression_eta: 0.0137 | Regression_pt: 0.2188 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4435 | Classification_binary: 3.8541 | Regression_cos_phi: 0.0990 | Regression_energy: 2.5139 | Regression_eta: 0.1086 | Regression_pt: 3.0426 | Regression_sin_phi: 0.1001 | LR: 4.07e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7576s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0855 | max_tensorboard_logging: 0.2124 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2748.7 | uss_MB: 1494.6 | child_rss_MB: 22916.6 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:29:02,928] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:29:02,977] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 05:29:02,978] INFO : Running validation on rank0 +rank0 [2026-11-14 05:29:02,978] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:29:02,978] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:29:03,079] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 05:29:03,161] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:29:03,239] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:29:03,289] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:29:03,335] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:29:03,381] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:29:03,428] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:29:03,475] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:29:03,552] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:29:03,629] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:29:03,706] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:29:03,754] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:29:03,799] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:29:03,845] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:29:03,892] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:29:03,938] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:29:04,004] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:29:04,008] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:29:04,010] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:29:04,012] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:29:04,014] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:29:04,015] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:29:04,017] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:29:04,019] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:29:04,020] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:29:04,022] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:29:04,023] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:29:04,025] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:29:04,027] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:29:04,028] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:29:04,030] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:29:04,032] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:29:05,009] INFO : VALIDATION | Step=20000/500000 | Train Loss=10.1618 | Valid Loss=9.6373 | Train Loss Unweighted=4.3767 | Valid Loss Unweighted=4.3313 | Stale=0 | ETA=597.9m +rank0 [2026-11-14 05:29:05,009] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:29:05,127] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:29:05,137] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:29:05,204] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:29:05,714] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:29:05,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:29:06,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:29:06,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:29:06,368] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:29:06,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:29:06,685] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:29:06,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:29:07,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:29:07,168] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:29:07,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:29:07,504] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:29:07,666] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:29:07,826] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:29:07,989] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:29:08,134] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:29:08,218] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:29:08,226] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:29:08,610] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:29:08,846] INFO : Plotted number of elements +rank0 [2026-11-14 05:29:09,778] INFO : Plotted jets +rank0 [2026-11-14 05:29:10,475] INFO : Plotted jet ratio +rank0 [2026-11-14 05:29:27,977] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 05:29:27,978] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:09,144] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 05:30:09,144] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:30:24,566] INFO : Step 21000/500000 rank0 | Train Loss: 7.6211 | Train Loss Unweighted: 3.8977 | Train Loss Components: Classification: 0.0367 | Classification_binary: 3.5622 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1387 | Regression_eta: 0.0065 | Regression_pt: 0.1405 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4076 | Classification_binary: 3.5622 | Regression_cos_phi: 0.0708 | Regression_energy: 1.5037 | Regression_eta: 0.0511 | Regression_pt: 1.9535 | Regression_sin_phi: 0.0723 | LR: 4.28e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7733s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0024 | max_backward: 0.0260 | max_optimizer_step: 0.0853 | max_tensorboard_logging: 0.2248 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2777.5 | uss_MB: 1523.5 | child_rss_MB: 22916.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:30:50,042] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 05:30:50,042] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:31:30,285] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 05:31:30,285] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:31:36,899] INFO : Step 22000/500000 rank0 | Train Loss: 9.1272 | Train Loss Unweighted: 4.1516 | Train Loss Components: Classification: 0.0302 | Classification_binary: 3.7008 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1909 | Regression_eta: 0.0195 | Regression_pt: 0.1916 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3355 | Classification_binary: 3.7008 | Regression_cos_phi: 0.0926 | Regression_energy: 2.0695 | Regression_eta: 0.1541 | Regression_pt: 2.6648 | Regression_sin_phi: 0.1099 | LR: 4.50e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7676s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0258 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.2206 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2778.6 | uss_MB: 1524.6 | child_rss_MB: 22916.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:31:36,901] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:31:36,949] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 05:32:12,562] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 05:32:12,562] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:32:52,121] INFO : Step 23000/500000 rank0 | Train Loss: 8.7751 | Train Loss Unweighted: 4.1982 | Train Loss Components: Classification: 0.0311 | Classification_binary: 3.7870 | Regression_cos_phi: 0.0144 | Regression_energy: 0.1750 | Regression_eta: 0.0049 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3457 | Classification_binary: 3.7870 | Regression_cos_phi: 0.1593 | Regression_energy: 1.8970 | Regression_eta: 0.0390 | Regression_pt: 2.4247 | Regression_sin_phi: 0.1224 | LR: 4.73e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7952s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0024 | max_backward: 0.0270 | max_optimizer_step: 0.1175 | max_tensorboard_logging: 0.2160 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2780.6 | uss_MB: 1526.6 | child_rss_MB: 22916.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:32:55,222] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 05:32:55,222] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:33:37,147] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 05:33:37,147] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:34:07,293] INFO : Step 24000/500000 rank0 | Train Loss: 8.4818 | Train Loss Unweighted: 4.0741 | Train Loss Components: Classification: 0.0372 | Classification_binary: 3.6757 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1693 | Regression_eta: 0.0083 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4129 | Classification_binary: 3.6757 | Regression_cos_phi: 0.1075 | Regression_energy: 1.8355 | Regression_eta: 0.0654 | Regression_pt: 2.2685 | Regression_sin_phi: 0.1162 | LR: 4.97e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.8176s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0247 | max_loss: 0.0023 | max_backward: 0.0274 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2780.6 | uss_MB: 1526.6 | child_rss_MB: 22916.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:34:07,295] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:34:07,343] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 05:34:19,669] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 05:34:19,669] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:00,956] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 05:35:00,957] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:22,068] INFO : Step 25000/500000 rank0 | Train Loss: 8.1232 | Train Loss Unweighted: 3.9974 | Train Loss Components: Classification: 0.0371 | Classification_binary: 3.6264 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1551 | Regression_eta: 0.0045 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4112 | Classification_binary: 3.6264 | Regression_cos_phi: 0.1126 | Regression_energy: 1.6814 | Regression_eta: 0.0359 | Regression_pt: 2.1673 | Regression_sin_phi: 0.0881 | LR: 5.22e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7750s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0299 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2188 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2777.5 | uss_MB: 1523.6 | child_rss_MB: 22919.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 32.3 +rank0 [2026-11-14 05:35:43,577] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 05:35:43,577] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:36:25,387] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 05:36:25,388] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:36:37,127] INFO : Step 26000/500000 rank0 | Train Loss: 9.4706 | Train Loss Unweighted: 4.2128 | Train Loss Components: Classification: 0.0345 | Classification_binary: 3.7432 | Regression_cos_phi: 0.0064 | Regression_energy: 0.2042 | Regression_eta: 0.0067 | Regression_pt: 0.2108 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3826 | Classification_binary: 3.7432 | Regression_cos_phi: 0.0710 | Regression_energy: 2.2143 | Regression_eta: 0.0532 | Regression_pt: 2.9318 | Regression_sin_phi: 0.0745 | LR: 5.47e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7440s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0248 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.2105 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2778.7 | uss_MB: 1524.7 | child_rss_MB: 22917.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:36:37,129] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:36:37,176] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 05:37:07,790] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 05:37:07,790] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:49,081] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 05:37:49,081] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:51,561] INFO : Step 27000/500000 rank0 | Train Loss: 9.4683 | Train Loss Unweighted: 4.1193 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.6410 | Regression_cos_phi: 0.0113 | Regression_energy: 0.2089 | Regression_eta: 0.0074 | Regression_pt: 0.2131 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3342 | Classification_binary: 3.6410 | Regression_cos_phi: 0.1250 | Regression_energy: 2.2653 | Regression_eta: 0.0585 | Regression_pt: 2.9633 | Regression_sin_phi: 0.0811 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7691s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0033 | max_backward: 0.0273 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.2157 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.1 | uss_MB: 1527.2 | child_rss_MB: 22917.0 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:38:31,732] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 05:38:31,732] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:06,212] INFO : Step 28000/500000 rank0 | Train Loss: 8.9539 | Train Loss Unweighted: 4.0407 | Train Loss Components: Classification: 0.0323 | Classification_binary: 3.5991 | Regression_cos_phi: 0.0123 | Regression_energy: 0.1852 | Regression_eta: 0.0101 | Regression_pt: 0.1914 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3586 | Classification_binary: 3.5991 | Regression_cos_phi: 0.1357 | Regression_energy: 2.0079 | Regression_eta: 0.0797 | Regression_pt: 2.6618 | Regression_sin_phi: 0.1111 | LR: 6.01e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.8085s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0025 | max_backward: 0.0260 | max_optimizer_step: 0.0854 | max_tensorboard_logging: 0.2643 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.2 | uss_MB: 1527.3 | child_rss_MB: 22918.4 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:39:06,214] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:39:06,261] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 05:39:13,335] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 05:39:13,335] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:55,100] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 05:39:55,100] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:40:20,924] INFO : Step 29000/500000 rank0 | Train Loss: 9.4892 | Train Loss Unweighted: 4.0655 | Train Loss Components: Classification: 0.0328 | Classification_binary: 3.5802 | Regression_cos_phi: 0.0082 | Regression_energy: 0.2089 | Regression_eta: 0.0103 | Regression_pt: 0.2177 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3639 | Classification_binary: 3.5802 | Regression_cos_phi: 0.0902 | Regression_energy: 2.2643 | Regression_eta: 0.0815 | Regression_pt: 3.0286 | Regression_sin_phi: 0.0805 | LR: 6.29e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7792s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0025 | max_backward: 0.0269 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2155 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2781.5 | uss_MB: 1527.5 | child_rss_MB: 22918.1 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:40:37,517] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 05:40:37,517] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:19,220] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 05:41:19,220] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:36,195] INFO : Step 30000/500000 rank0 | Train Loss: 8.0177 | Train Loss Unweighted: 3.7923 | Train Loss Components: Classification: 0.0289 | Classification_binary: 3.4128 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1638 | Regression_eta: 0.0079 | Regression_pt: 0.1647 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3203 | Classification_binary: 3.4128 | Regression_cos_phi: 0.0663 | Regression_energy: 1.7761 | Regression_eta: 0.0624 | Regression_pt: 2.2912 | Regression_sin_phi: 0.0886 | LR: 6.58e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7774s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0273 | max_loss: 0.0023 | max_backward: 0.0274 | max_optimizer_step: 0.0871 | max_tensorboard_logging: 0.2234 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2778.6 | uss_MB: 1524.6 | child_rss_MB: 22921.7 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:41:36,197] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:41:36,245] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 05:41:36,246] INFO : Running validation on rank0 +rank0 [2026-11-14 05:41:36,246] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:41:36,246] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:41:36,349] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 05:41:36,430] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:41:36,509] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:41:36,557] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:41:36,602] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:41:36,648] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:41:36,694] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:41:36,740] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:41:36,804] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:41:36,882] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:41:36,959] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:41:37,005] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:41:37,050] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:41:37,097] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:41:37,145] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:41:37,192] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:41:37,258] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:41:37,262] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:41:37,264] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:41:37,266] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:41:37,267] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:41:37,269] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:41:37,270] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:41:37,272] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:41:37,274] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:41:37,275] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:41:37,277] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:41:37,278] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:41:37,280] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:41:37,281] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:41:37,283] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:41:37,285] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:41:38,268] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.0177 | Valid Loss=9.3713 | Train Loss Unweighted=3.7923 | Valid Loss Unweighted=4.1966 | Stale=0 | ETA=587.0m +rank0 [2026-11-14 05:41:38,268] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:41:38,365] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:41:38,375] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:41:38,426] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:41:38,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:41:39,040] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:41:39,203] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:41:39,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:41:39,525] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:41:39,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:41:39,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:41:40,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:41:40,170] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:41:40,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:41:40,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:41:40,651] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:41:40,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:41:40,977] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:41:41,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:41:41,298] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:41:41,420] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:41:41,427] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:41:41,820] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:41:42,059] INFO : Plotted number of elements +rank0 [2026-11-14 05:41:42,984] INFO : Plotted jets +rank0 [2026-11-14 05:41:43,684] INFO : Plotted jet ratio +rank0 [2026-11-14 05:42:10,080] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 05:42:10,080] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:42:51,933] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 05:42:51,933] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:42:59,329] INFO : Step 31000/500000 rank0 | Train Loss: 9.2547 | Train Loss Unweighted: 4.0277 | Train Loss Components: Classification: 0.0323 | Classification_binary: 3.5584 | Regression_cos_phi: 0.0104 | Regression_energy: 0.2013 | Regression_eta: 0.0082 | Regression_pt: 0.2031 | Regression_sin_phi: 0.0140 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3582 | Classification_binary: 3.5584 | Regression_cos_phi: 0.1150 | Regression_energy: 2.1821 | Regression_eta: 0.0652 | Regression_pt: 2.8248 | Regression_sin_phi: 0.1510 | LR: 6.88e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7772s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.2215 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2906.9 | uss_MB: 1653.0 | child_rss_MB: 22916.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:43:34,365] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 05:43:34,366] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:14,417] INFO : Step 32000/500000 rank0 | Train Loss: 10.3134 | Train Loss Unweighted: 4.3098 | Train Loss Components: Classification: 0.0343 | Classification_binary: 3.7697 | Regression_cos_phi: 0.0199 | Regression_energy: 0.2235 | Regression_eta: 0.0135 | Regression_pt: 0.2334 | Regression_sin_phi: 0.0156 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3811 | Classification_binary: 3.7697 | Regression_cos_phi: 0.2195 | Regression_energy: 2.4232 | Regression_eta: 0.1065 | Regression_pt: 3.2461 | Regression_sin_phi: 0.1673 | LR: 7.19e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.8172s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0004 | max_forward: 0.0246 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0878 | max_tensorboard_logging: 0.2232 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2907.0 | uss_MB: 1653.0 | child_rss_MB: 22916.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:44:14,419] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:44:14,467] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 05:44:17,004] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 05:44:17,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:59,120] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 05:44:59,121] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:45:29,840] INFO : Step 33000/500000 rank0 | Train Loss: 7.5124 | Train Loss Unweighted: 3.6857 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.3412 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1491 | Regression_eta: 0.0082 | Regression_pt: 0.1478 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2821 | Classification_binary: 3.3412 | Regression_cos_phi: 0.0841 | Regression_energy: 1.6163 | Regression_eta: 0.0646 | Regression_pt: 2.0562 | Regression_sin_phi: 0.0680 | LR: 7.51e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.7902s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0247 | max_loss: 0.0025 | max_backward: 0.0272 | max_optimizer_step: 0.0875 | max_tensorboard_logging: 0.2213 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2803.7 | uss_MB: 1549.7 | child_rss_MB: 22916.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:45:41,579] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 05:45:41,579] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:23,313] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 05:46:23,313] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:44,788] INFO : Step 34000/500000 rank0 | Train Loss: 8.2648 | Train Loss Unweighted: 3.9463 | Train Loss Components: Classification: 0.0328 | Classification_binary: 3.5580 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1630 | Regression_eta: 0.0078 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3643 | Classification_binary: 3.5580 | Regression_cos_phi: 0.0986 | Regression_energy: 1.7673 | Regression_eta: 0.0613 | Regression_pt: 2.3144 | Regression_sin_phi: 0.1009 | LR: 7.83e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.7770s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0246 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2216 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2804.4 | uss_MB: 1550.5 | child_rss_MB: 22917.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:46:44,790] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:46:44,837] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 05:47:05,824] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 05:47:05,825] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:47:47,598] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 05:47:47,598] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:00,210] INFO : Step 35000/500000 rank0 | Train Loss: 7.6021 | Train Loss Unweighted: 3.7834 | Train Loss Components: Classification: 0.0355 | Classification_binary: 3.4405 | Regression_cos_phi: 0.0044 | Regression_energy: 0.1451 | Regression_eta: 0.0046 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3941 | Classification_binary: 3.4405 | Regression_cos_phi: 0.0481 | Regression_energy: 1.5736 | Regression_eta: 0.0364 | Regression_pt: 2.0335 | Regression_sin_phi: 0.0758 | LR: 8.16e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.7628s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0024 | max_backward: 0.0272 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2139 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.7 | uss_MB: 1552.8 | child_rss_MB: 22916.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:48:30,345] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 05:48:30,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:49:12,077] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 05:49:12,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:49:15,137] INFO : Step 36000/500000 rank0 | Train Loss: 9.2753 | Train Loss Unweighted: 4.0148 | Train Loss Components: Classification: 0.0296 | Classification_binary: 3.5439 | Regression_cos_phi: 0.0049 | Regression_energy: 0.2049 | Regression_eta: 0.0104 | Regression_pt: 0.2113 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3289 | Classification_binary: 3.5439 | Regression_cos_phi: 0.0542 | Regression_energy: 2.2209 | Regression_eta: 0.0821 | Regression_pt: 2.9395 | Regression_sin_phi: 0.1057 | LR: 8.50e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.7657s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.2117 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.8 | uss_MB: 1552.9 | child_rss_MB: 22915.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:49:15,138] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:49:15,185] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 05:49:54,536] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 05:49:54,536] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:30,036] INFO : Step 37000/500000 rank0 | Train Loss: 8.2987 | Train Loss Unweighted: 3.8319 | Train Loss Components: Classification: 0.0336 | Classification_binary: 3.4297 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1728 | Regression_eta: 0.0079 | Regression_pt: 0.1703 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3729 | Classification_binary: 3.4297 | Regression_cos_phi: 0.0983 | Regression_energy: 1.8733 | Regression_eta: 0.0626 | Regression_pt: 2.3680 | Regression_sin_phi: 0.0939 | LR: 8.85e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7736s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0278 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2183 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.8 | uss_MB: 1552.9 | child_rss_MB: 22917.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:50:36,833] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 05:50:36,833] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:18,789] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 05:51:18,789] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:45,241] INFO : Step 38000/500000 rank0 | Train Loss: 7.6105 | Train Loss Unweighted: 3.7278 | Train Loss Components: Classification: 0.0366 | Classification_binary: 3.3744 | Regression_cos_phi: 0.0120 | Regression_energy: 0.1404 | Regression_eta: 0.0115 | Regression_pt: 0.1393 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4057 | Classification_binary: 3.3744 | Regression_cos_phi: 0.1330 | Regression_energy: 1.5224 | Regression_eta: 0.0910 | Regression_pt: 1.9376 | Regression_sin_phi: 0.1463 | LR: 9.21e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.7730s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0247 | max_loss: 0.0023 | max_backward: 0.0274 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.2184 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2804.5 | uss_MB: 1550.6 | child_rss_MB: 22918.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:51:45,242] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:51:45,288] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 05:52:01,381] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 05:52:01,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:43,173] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 05:52:43,173] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:59,999] INFO : Step 39000/500000 rank0 | Train Loss: 7.7061 | Train Loss Unweighted: 3.6005 | Train Loss Components: Classification: 0.0296 | Classification_binary: 3.2299 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1547 | Regression_eta: 0.0108 | Regression_pt: 0.1567 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3280 | Classification_binary: 3.2299 | Regression_cos_phi: 0.1131 | Regression_energy: 1.6770 | Regression_eta: 0.0853 | Regression_pt: 2.1798 | Regression_sin_phi: 0.0930 | LR: 9.57e-05 | DataLoad Time: 0.0013s | Model Forward Time: 0.7704s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0276 | max_optimizer_step: 0.0863 | max_tensorboard_logging: 0.2201 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2806.9 | uss_MB: 1553.0 | child_rss_MB: 22918.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:53:24,752] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 05:53:24,752] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:05,795] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 05:54:05,795] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:13,537] INFO : Step 40000/500000 rank0 | Train Loss: 7.9346 | Train Loss Unweighted: 3.7324 | Train Loss Components: Classification: 0.0317 | Classification_binary: 3.3526 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1576 | Regression_eta: 0.0104 | Regression_pt: 0.1584 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3515 | Classification_binary: 3.3526 | Regression_cos_phi: 0.0920 | Regression_energy: 1.7088 | Regression_eta: 0.0821 | Regression_pt: 2.2035 | Regression_sin_phi: 0.1440 | LR: 9.94e-05 | DataLoad Time: 0.0093s | Model Forward Time: 0.7691s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0275 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2170 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2807.3 | uss_MB: 1553.4 | child_rss_MB: 22918.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:54:13,539] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 05:54:13,587] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 05:54:13,588] INFO : Running validation on rank0 +rank0 [2026-11-14 05:54:13,588] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:54:13,588] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:54:13,692] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 05:54:13,776] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:54:13,854] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:54:13,905] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:54:13,951] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:54:13,996] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:54:14,042] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:54:14,088] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:54:14,165] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:54:14,245] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:54:14,323] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:54:14,370] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:54:14,417] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:54:14,463] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:54:14,510] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:54:14,556] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:54:14,634] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:54:14,640] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:54:14,641] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:54:14,643] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:54:14,645] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:54:14,646] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:54:14,648] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:54:14,649] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:54:14,651] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:54:14,653] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:54:14,654] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:54:14,656] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:54:14,657] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:54:14,659] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:54:14,661] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:54:14,662] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:54:15,648] INFO : VALIDATION | Step=40000/500000 | Train Loss=7.9346 | Valid Loss=8.8835 | Train Loss Unweighted=3.7324 | Valid Loss Unweighted=3.9793 | Stale=0 | ETA=576.0m +rank0 [2026-11-14 05:54:15,648] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:54:15,750] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:54:15,759] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:54:15,810] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:54:16,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:54:16,426] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:54:16,590] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:54:16,753] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:54:16,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:54:17,076] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:54:17,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:54:17,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:54:17,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:54:17,722] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:54:17,881] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:54:18,044] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:54:18,216] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:54:18,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:54:18,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:54:18,687] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:54:18,791] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 05:54:18,798] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:54:19,173] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:54:19,411] INFO : Plotted number of elements +rank0 [2026-11-14 05:54:20,349] INFO : Plotted jets +rank0 [2026-11-14 05:54:21,044] INFO : Plotted jet ratio +rank0 [2026-11-14 05:54:55,797] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 05:54:55,797] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:55:36,043] INFO : Step 41000/500000 rank0 | Train Loss: 7.2046 | Train Loss Unweighted: 3.5449 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.2137 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1318 | Regression_eta: 0.0111 | Regression_pt: 0.1381 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3592 | Classification_binary: 3.2137 | Regression_cos_phi: 0.0921 | Regression_energy: 1.4293 | Regression_eta: 0.0878 | Regression_pt: 1.9203 | Regression_sin_phi: 0.1022 | LR: 1.03e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.8240s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.1345 | max_tensorboard_logging: 0.2189 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2828.7 | uss_MB: 1574.8 | child_rss_MB: 22919.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:55:37,963] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 05:55:37,963] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:19,323] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 05:56:19,323] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:50,502] INFO : Step 42000/500000 rank0 | Train Loss: 7.6894 | Train Loss Unweighted: 3.7740 | Train Loss Components: Classification: 0.0331 | Classification_binary: 3.4203 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1418 | Regression_eta: 0.0069 | Regression_pt: 0.1454 | Regression_sin_phi: 0.0173 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3675 | Classification_binary: 3.4203 | Regression_cos_phi: 0.1015 | Regression_energy: 1.5374 | Regression_eta: 0.0548 | Regression_pt: 2.0222 | Regression_sin_phi: 0.1857 | LR: 1.07e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7589s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0851 | max_tensorboard_logging: 0.2135 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2829.4 | uss_MB: 1575.6 | child_rss_MB: 22917.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:56:50,505] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 05:56:50,552] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 05:57:01,538] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 05:57:01,538] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:57:43,057] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 05:57:43,057] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:04,933] INFO : Step 43000/500000 rank0 | Train Loss: 7.5353 | Train Loss Unweighted: 3.7548 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.4150 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1415 | Regression_eta: 0.0070 | Regression_pt: 0.1463 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2906 | Classification_binary: 3.4150 | Regression_cos_phi: 0.0990 | Regression_energy: 1.5345 | Regression_eta: 0.0554 | Regression_pt: 2.0344 | Regression_sin_phi: 0.1064 | LR: 1.11e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7540s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0260 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2066 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2831.5 | uss_MB: 1577.7 | child_rss_MB: 22919.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:58:24,654] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 05:58:24,654] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:59:06,165] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 05:59:06,166] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:59:19,057] INFO : Step 44000/500000 rank0 | Train Loss: 7.9219 | Train Loss Unweighted: 3.6805 | Train Loss Components: Classification: 0.0352 | Classification_binary: 3.2998 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1627 | Regression_eta: 0.0049 | Regression_pt: 0.1629 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3910 | Classification_binary: 3.2998 | Regression_cos_phi: 0.0826 | Regression_energy: 1.7637 | Regression_eta: 0.0389 | Regression_pt: 2.2655 | Regression_sin_phi: 0.0803 | LR: 1.15e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7726s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2196 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2832.0 | uss_MB: 1578.1 | child_rss_MB: 22919.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 24.3 +rank0 [2026-11-14 05:59:19,059] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 05:59:19,105] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 05:59:48,651] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 05:59:48,651] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:29,621] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 06:00:29,621] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:33,275] INFO : Step 45000/500000 rank0 | Train Loss: 7.7389 | Train Loss Unweighted: 3.7186 | Train Loss Components: Classification: 0.0291 | Classification_binary: 3.3569 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1547 | Regression_eta: 0.0077 | Regression_pt: 0.1548 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3231 | Classification_binary: 3.3569 | Regression_cos_phi: 0.0644 | Regression_energy: 1.6773 | Regression_eta: 0.0609 | Regression_pt: 2.1534 | Regression_sin_phi: 0.1029 | LR: 1.19e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7621s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2106 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2832.1 | uss_MB: 1578.2 | child_rss_MB: 22920.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:01:11,691] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 06:01:11,691] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:47,398] INFO : Step 46000/500000 rank0 | Train Loss: 7.6446 | Train Loss Unweighted: 3.7153 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.3588 | Regression_cos_phi: 0.0121 | Regression_energy: 0.1505 | Regression_eta: 0.0108 | Regression_pt: 0.1446 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3298 | Classification_binary: 3.3588 | Regression_cos_phi: 0.1338 | Regression_energy: 1.6312 | Regression_eta: 0.0852 | Regression_pt: 2.0105 | Regression_sin_phi: 0.0953 | LR: 1.23e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7584s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0262 | max_loss: 0.0025 | max_backward: 0.0261 | max_optimizer_step: 0.0850 | max_tensorboard_logging: 0.2162 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2829.6 | uss_MB: 1575.7 | child_rss_MB: 22918.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:01:47,400] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 06:01:47,447] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 06:01:53,591] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 06:01:53,591] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:35,236] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 06:02:35,236] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:02,221] INFO : Step 47000/500000 rank0 | Train Loss: 7.6868 | Train Loss Unweighted: 3.5246 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.1522 | Regression_cos_phi: 0.0047 | Regression_energy: 0.1645 | Regression_eta: 0.0053 | Regression_pt: 0.1646 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3178 | Classification_binary: 3.1522 | Regression_cos_phi: 0.0522 | Regression_energy: 1.7830 | Regression_eta: 0.0422 | Regression_pt: 2.2889 | Regression_sin_phi: 0.0505 | LR: 1.27e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7753s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0024 | max_backward: 0.0272 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2182 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2831.6 | uss_MB: 1577.8 | child_rss_MB: 22918.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:03:17,626] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 06:03:17,626] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:59,910] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 06:03:59,910] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:17,755] INFO : Step 48000/500000 rank0 | Train Loss: 7.6529 | Train Loss Unweighted: 3.5439 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.1705 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1575 | Regression_eta: 0.0129 | Regression_pt: 0.1510 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3276 | Classification_binary: 3.1705 | Regression_cos_phi: 0.1169 | Regression_energy: 1.7073 | Regression_eta: 0.1017 | Regression_pt: 2.1004 | Regression_sin_phi: 0.1285 | LR: 1.31e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7664s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2169 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2831.7 | uss_MB: 1577.8 | child_rss_MB: 22919.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:04:17,756] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 06:04:17,804] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:04:42,402] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 06:04:42,403] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:24,042] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 06:05:24,042] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:32,594] INFO : Step 49000/500000 rank0 | Train Loss: 8.0698 | Train Loss Unweighted: 3.7802 | Train Loss Components: Classification: 0.0343 | Classification_binary: 3.3932 | Regression_cos_phi: 0.0159 | Regression_energy: 0.1550 | Regression_eta: 0.0077 | Regression_pt: 0.1607 | Regression_sin_phi: 0.0134 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3809 | Classification_binary: 3.3932 | Regression_cos_phi: 0.1753 | Regression_energy: 1.6803 | Regression_eta: 0.0609 | Regression_pt: 2.2356 | Regression_sin_phi: 0.1436 | LR: 1.36e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7656s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0268 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2149 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2831.7 | uss_MB: 1577.9 | child_rss_MB: 22918.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:06,328] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 06:06:06,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:06:47,458] INFO : Step 50000/500000 rank0 | Train Loss: 7.9159 | Train Loss Unweighted: 3.7908 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.4212 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1631 | Regression_eta: 0.0050 | Regression_pt: 0.1613 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3063 | Classification_binary: 3.4212 | Regression_cos_phi: 0.0662 | Regression_energy: 1.7680 | Regression_eta: 0.0393 | Regression_pt: 2.2437 | Regression_sin_phi: 0.0713 | LR: 1.40e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7668s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0260 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.2173 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2828.9 | uss_MB: 1575.0 | child_rss_MB: 22919.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:47,460] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 06:06:47,506] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:06:47,506] INFO : Running validation on rank0 +rank0 [2026-11-14 06:06:47,506] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:06:47,506] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:06:47,629] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 06:06:47,712] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:06:47,792] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:06:47,842] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:06:47,889] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:06:47,935] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:06:47,982] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:06:48,029] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:06:48,097] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:06:48,175] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:06:48,253] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:06:48,300] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:06:48,346] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:06:48,393] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:06:48,440] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:06:48,491] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:06:48,562] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:06:48,567] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:06:48,569] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:06:48,570] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:06:48,572] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:06:48,574] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:06:48,575] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:06:48,577] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:06:48,579] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:06:48,580] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:06:48,582] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:06:48,583] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:06:48,585] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:06:48,587] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:06:48,588] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:06:48,590] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:06:49,580] INFO : VALIDATION | Step=50000/500000 | Train Loss=7.9159 | Valid Loss=8.7096 | Train Loss Unweighted=3.7908 | Valid Loss Unweighted=3.9263 | Stale=0 | ETA=563.9m +rank0 [2026-11-14 06:06:49,581] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:06:49,698] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:06:49,707] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:06:49,773] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:06:50,294] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:06:50,459] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:06:50,622] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:06:50,784] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:06:50,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:06:51,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:06:51,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:06:51,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:06:51,582] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:06:51,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:06:51,919] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:06:52,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:06:52,243] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:06:52,403] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:06:52,564] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:06:52,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:06:52,817] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:06:52,824] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:06:53,207] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:06:53,446] INFO : Plotted number of elements +rank0 [2026-11-14 06:06:54,380] INFO : Plotted jets +rank0 [2026-11-14 06:06:55,081] INFO : Plotted jet ratio +rank0 [2026-11-14 06:06:56,836] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 06:06:56,836] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:38,647] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 06:07:38,648] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:08:10,683] INFO : Step 51000/500000 rank0 | Train Loss: 6.2996 | Train Loss Unweighted: 3.2629 | Train Loss Components: Classification: 0.0270 | Classification_binary: 2.9900 | Regression_cos_phi: 0.0034 | Regression_energy: 0.1087 | Regression_eta: 0.0059 | Regression_pt: 0.1178 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2993 | Classification_binary: 2.9900 | Regression_cos_phi: 0.0377 | Regression_energy: 1.1788 | Regression_eta: 0.0469 | Regression_pt: 1.6391 | Regression_sin_phi: 0.1078 | LR: 1.44e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7786s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0025 | max_backward: 0.0273 | max_optimizer_step: 0.0870 | max_tensorboard_logging: 0.2180 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2962.6 | uss_MB: 1708.7 | child_rss_MB: 22919.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:08:21,235] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 06:08:21,235] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:03,118] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 06:09:03,118] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:25,793] INFO : Step 52000/500000 rank0 | Train Loss: 7.4622 | Train Loss Unweighted: 3.3252 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.9541 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1613 | Regression_eta: 0.0080 | Regression_pt: 0.1631 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2500 | Classification_binary: 2.9541 | Regression_cos_phi: 0.1181 | Regression_energy: 1.7487 | Regression_eta: 0.0636 | Regression_pt: 2.2686 | Regression_sin_phi: 0.0592 | LR: 1.49e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7676s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0240 | max_loss: 0.0026 | max_backward: 0.0273 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2127 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2962.8 | uss_MB: 1709.0 | child_rss_MB: 22918.2 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:09:25,795] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 06:09:25,842] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 06:09:45,590] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 06:09:45,590] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:10:27,454] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 06:10:27,454] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:10:40,904] INFO : Step 53000/500000 rank0 | Train Loss: 7.8325 | Train Loss Unweighted: 3.6036 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.2239 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1624 | Regression_eta: 0.0066 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3267 | Classification_binary: 3.2239 | Regression_cos_phi: 0.1215 | Regression_energy: 1.7604 | Regression_eta: 0.0519 | Regression_pt: 2.2773 | Regression_sin_phi: 0.0709 | LR: 1.53e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7752s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0023 | max_backward: 0.0269 | max_optimizer_step: 0.0864 | max_tensorboard_logging: 0.2172 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2962.6 | uss_MB: 1708.8 | child_rss_MB: 22920.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:11:09,811] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 06:11:09,811] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:53,513] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 06:11:53,515] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:57,821] INFO : Step 54000/500000 rank0 | Train Loss: 7.4899 | Train Loss Unweighted: 3.4385 | Train Loss Components: Classification: 0.0243 | Classification_binary: 3.0723 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1550 | Regression_eta: 0.0116 | Regression_pt: 0.1546 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2694 | Classification_binary: 3.0723 | Regression_cos_phi: 0.1315 | Regression_energy: 1.6804 | Regression_eta: 0.0917 | Regression_pt: 2.1502 | Regression_sin_phi: 0.0944 | LR: 1.58e-04 | DataLoad Time: 0.0163s | Model Forward Time: 0.7880s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0250 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0896 | max_tensorboard_logging: 0.2396 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2856.2 | uss_MB: 1602.4 | child_rss_MB: 22919.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:11:57,823] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 06:11:57,904] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 06:12:36,129] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 06:12:36,129] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:13,323] INFO : Step 55000/500000 rank0 | Train Loss: 7.1302 | Train Loss Unweighted: 3.6363 | Train Loss Components: Classification: 0.0361 | Classification_binary: 3.3210 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1284 | Regression_eta: 0.0077 | Regression_pt: 0.1317 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.4010 | Classification_binary: 3.3210 | Regression_cos_phi: 0.0546 | Regression_energy: 1.3917 | Regression_eta: 0.0607 | Regression_pt: 1.8312 | Regression_sin_phi: 0.0700 | LR: 1.62e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7705s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0239 | max_loss: 0.0024 | max_backward: 0.0270 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.2203 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2859.6 | uss_MB: 1605.8 | child_rss_MB: 22921.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:13:18,950] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 06:13:18,950] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:14:00,798] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 06:14:00,798] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:14:28,421] INFO : Step 56000/500000 rank0 | Train Loss: 5.5289 | Train Loss Unweighted: 3.0077 | Train Loss Components: Classification: 0.0254 | Classification_binary: 2.7794 | Regression_cos_phi: 0.0039 | Regression_energy: 0.0946 | Regression_eta: 0.0059 | Regression_pt: 0.0927 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2819 | Classification_binary: 2.7794 | Regression_cos_phi: 0.0427 | Regression_energy: 1.0254 | Regression_eta: 0.0465 | Regression_pt: 1.2893 | Regression_sin_phi: 0.0637 | LR: 1.67e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7813s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0865 | max_tensorboard_logging: 0.2348 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2860.1 | uss_MB: 1606.3 | child_rss_MB: 22922.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:14:28,422] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 06:14:28,472] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 06:14:43,398] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 06:14:43,398] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:25,239] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 06:15:25,239] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:43,665] INFO : Step 57000/500000 rank0 | Train Loss: 6.6769 | Train Loss Unweighted: 3.4412 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.1476 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1157 | Regression_eta: 0.0123 | Regression_pt: 0.1222 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3018 | Classification_binary: 3.1476 | Regression_cos_phi: 0.0974 | Regression_energy: 1.2541 | Regression_eta: 0.0976 | Regression_pt: 1.7002 | Regression_sin_phi: 0.0782 | LR: 1.72e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7801s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0314 | max_optimizer_step: 0.0871 | max_tensorboard_logging: 0.2198 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2861.2 | uss_MB: 1607.3 | child_rss_MB: 22922.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:16:07,774] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 06:16:07,775] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:16:49,940] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 06:16:49,940] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:16:59,097] INFO : Step 58000/500000 rank0 | Train Loss: 6.6022 | Train Loss Unweighted: 3.6380 | Train Loss Components: Classification: 0.0354 | Classification_binary: 3.3687 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1049 | Regression_eta: 0.0061 | Regression_pt: 0.1048 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3928 | Classification_binary: 3.3687 | Regression_cos_phi: 0.1116 | Regression_energy: 1.1369 | Regression_eta: 0.0481 | Regression_pt: 1.4577 | Regression_sin_phi: 0.0863 | LR: 1.76e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.8059s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0024 | max_backward: 0.0282 | max_optimizer_step: 0.0878 | max_tensorboard_logging: 0.2202 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.5 | uss_MB: 1601.7 | child_rss_MB: 22920.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:16:59,098] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 06:16:59,149] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 06:17:32,372] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 06:17:32,372] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:12,971] INFO : Step 59000/500000 rank0 | Train Loss: 7.0903 | Train Loss Unweighted: 3.4728 | Train Loss Components: Classification: 0.0349 | Classification_binary: 3.1450 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1325 | Regression_eta: 0.0099 | Regression_pt: 0.1339 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3872 | Classification_binary: 3.1450 | Regression_cos_phi: 0.0936 | Regression_energy: 1.4368 | Regression_eta: 0.0783 | Regression_pt: 1.8620 | Regression_sin_phi: 0.0874 | LR: 1.81e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7766s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0023 | max_backward: 0.0263 | max_optimizer_step: 0.0871 | max_tensorboard_logging: 0.2206 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2860.1 | uss_MB: 1606.3 | child_rss_MB: 22921.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:18:13,694] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 06:18:13,694] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:54,369] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 06:18:54,369] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:19:27,194] INFO : Step 60000/500000 rank0 | Train Loss: 6.6738 | Train Loss Unweighted: 3.4087 | Train Loss Components: Classification: 0.0272 | Classification_binary: 3.1107 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1233 | Regression_eta: 0.0118 | Regression_pt: 0.1169 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3017 | Classification_binary: 3.1107 | Regression_cos_phi: 0.1085 | Regression_energy: 1.3365 | Regression_eta: 0.0933 | Regression_pt: 1.6258 | Regression_sin_phi: 0.0974 | LR: 1.86e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7709s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2198 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2860.2 | uss_MB: 1606.4 | child_rss_MB: 22914.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:19:27,196] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 06:19:27,248] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 06:19:27,249] INFO : Running validation on rank0 +rank0 [2026-11-14 06:19:27,249] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:19:27,249] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:19:27,355] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 06:19:27,437] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:19:27,536] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:19:27,587] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:19:27,633] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:19:27,679] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:19:27,726] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:19:27,772] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:19:27,837] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:19:27,915] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:19:27,992] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:19:28,039] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:19:28,084] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:19:28,130] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:19:28,176] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:19:28,222] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:19:28,291] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:19:28,296] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:19:28,297] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:19:28,299] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:19:28,300] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:19:28,302] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:19:28,304] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:19:28,306] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:19:28,307] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:19:28,310] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:19:28,311] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:19:28,312] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:19:28,313] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:19:28,315] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:19:28,316] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:19:28,318] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:19:29,292] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.6738 | Valid Loss=8.9453 | Train Loss Unweighted=3.4087 | Valid Loss Unweighted=3.9658 | Stale=1 | ETA=552.3m +rank0 [2026-11-14 06:19:29,292] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:19:29,395] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:19:29,403] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:19:29,454] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:19:29,896] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:19:30,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:19:30,222] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:19:30,395] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:19:30,557] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:19:30,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:19:30,880] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:19:31,041] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:19:31,202] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:19:31,365] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:19:31,529] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:19:31,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:19:31,849] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:19:32,006] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:19:32,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:19:32,311] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:19:32,405] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:19:32,429] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:19:32,816] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:19:33,063] INFO : Plotted number of elements +rank0 [2026-11-14 06:19:33,989] INFO : Plotted jets +rank0 [2026-11-14 06:19:34,678] INFO : Plotted jet ratio +rank0 [2026-11-14 06:19:45,038] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 06:19:45,038] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:26,490] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 06:20:26,490] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:49,097] INFO : Step 61000/500000 rank0 | Train Loss: 6.0582 | Train Loss Unweighted: 3.0949 | Train Loss Components: Classification: 0.0252 | Classification_binary: 2.8267 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1091 | Regression_eta: 0.0067 | Regression_pt: 0.1097 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2793 | Classification_binary: 2.8267 | Regression_cos_phi: 0.0886 | Regression_energy: 1.1826 | Regression_eta: 0.0531 | Regression_pt: 1.5253 | Regression_sin_phi: 0.1024 | LR: 1.91e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7888s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0326 | max_optimizer_step: 0.0874 | max_tensorboard_logging: 0.2170 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2991.5 | uss_MB: 1737.6 | child_rss_MB: 22912.5 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:21:08,228] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 06:21:08,228] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:21:49,945] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 06:21:49,945] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:04,037] INFO : Step 62000/500000 rank0 | Train Loss: 6.4799 | Train Loss Unweighted: 3.3760 | Train Loss Components: Classification: 0.0217 | Classification_binary: 3.0985 | Regression_cos_phi: 0.0032 | Regression_energy: 0.1223 | Regression_eta: 0.0037 | Regression_pt: 0.1231 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2411 | Classification_binary: 3.0985 | Regression_cos_phi: 0.0353 | Regression_energy: 1.3264 | Regression_eta: 0.0292 | Regression_pt: 1.7117 | Regression_sin_phi: 0.0378 | LR: 1.95e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.8108s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0272 | max_optimizer_step: 0.1284 | max_tensorboard_logging: 0.2181 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2883.7 | uss_MB: 1629.9 | child_rss_MB: 22910.8 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:22:04,039] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 06:22:04,091] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 06:22:32,403] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 06:22:32,403] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:13,548] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 06:23:13,548] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:18,291] INFO : Step 63000/500000 rank0 | Train Loss: 6.5941 | Train Loss Unweighted: 3.2361 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.9347 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1298 | Regression_eta: 0.0046 | Regression_pt: 0.1303 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2471 | Classification_binary: 2.9347 | Regression_cos_phi: 0.0698 | Regression_energy: 1.4076 | Regression_eta: 0.0367 | Regression_pt: 1.8116 | Regression_sin_phi: 0.0866 | LR: 2.00e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7743s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0245 | max_loss: 0.0024 | max_backward: 0.0259 | max_optimizer_step: 0.0856 | max_tensorboard_logging: 0.2196 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2887.5 | uss_MB: 1633.7 | child_rss_MB: 22913.9 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:23:55,706] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 06:23:55,706] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:33,061] INFO : Step 64000/500000 rank0 | Train Loss: 5.9262 | Train Loss Unweighted: 3.2268 | Train Loss Components: Classification: 0.0230 | Classification_binary: 2.9823 | Regression_cos_phi: 0.0070 | Regression_energy: 0.0991 | Regression_eta: 0.0076 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2557 | Classification_binary: 2.9823 | Regression_cos_phi: 0.0769 | Regression_energy: 1.0745 | Regression_eta: 0.0601 | Regression_pt: 1.4012 | Regression_sin_phi: 0.0755 | LR: 2.05e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7738s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0025 | max_backward: 0.0272 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2130 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2887.6 | uss_MB: 1633.8 | child_rss_MB: 22911.9 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:24:33,062] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 06:24:33,110] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 06:24:38,156] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 06:24:38,157] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:19,994] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 06:25:19,994] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:48,555] INFO : Step 65000/500000 rank0 | Train Loss: 5.4422 | Train Loss Unweighted: 3.0169 | Train Loss Components: Classification: 0.0206 | Classification_binary: 2.7971 | Regression_cos_phi: 0.0047 | Regression_energy: 0.0873 | Regression_eta: 0.0085 | Regression_pt: 0.0918 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2284 | Classification_binary: 2.7971 | Regression_cos_phi: 0.0518 | Regression_energy: 0.9466 | Regression_eta: 0.0673 | Regression_pt: 1.2775 | Regression_sin_phi: 0.0736 | LR: 2.10e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.7666s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0024 | max_backward: 0.0273 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2135 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2887.8 | uss_MB: 1633.9 | child_rss_MB: 22912.7 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:26:02,767] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 06:26:02,767] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:44,512] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 06:26:44,512] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:27:03,494] INFO : Step 66000/500000 rank0 | Train Loss: 5.6695 | Train Loss Unweighted: 2.9567 | Train Loss Components: Classification: 0.0256 | Classification_binary: 2.7120 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1014 | Regression_eta: 0.0051 | Regression_pt: 0.1018 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2836 | Classification_binary: 2.7120 | Regression_cos_phi: 0.0582 | Regression_energy: 1.0991 | Regression_eta: 0.0401 | Regression_pt: 1.4159 | Regression_sin_phi: 0.0606 | LR: 2.15e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7604s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0024 | max_backward: 0.0272 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2165 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2883.1 | uss_MB: 1629.3 | child_rss_MB: 22912.6 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:27:03,496] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 06:27:03,548] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 06:27:27,332] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 06:27:27,332] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:08,975] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 06:28:08,975] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:18,789] INFO : Step 67000/500000 rank0 | Train Loss: 5.3649 | Train Loss Unweighted: 2.9360 | Train Loss Components: Classification: 0.0234 | Classification_binary: 2.7167 | Regression_cos_phi: 0.0051 | Regression_energy: 0.0903 | Regression_eta: 0.0052 | Regression_pt: 0.0909 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2598 | Classification_binary: 2.7167 | Regression_cos_phi: 0.0564 | Regression_energy: 0.9788 | Regression_eta: 0.0411 | Regression_pt: 1.2636 | Regression_sin_phi: 0.0486 | LR: 2.20e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.8091s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0879 | max_tensorboard_logging: 0.2122 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2886.5 | uss_MB: 1632.7 | child_rss_MB: 22913.4 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:28:51,860] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 06:28:51,860] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:29:34,047] INFO : Step 68000/500000 rank0 | Train Loss: 5.3894 | Train Loss Unweighted: 3.0968 | Train Loss Components: Classification: 0.0304 | Classification_binary: 2.8877 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0839 | Regression_eta: 0.0064 | Regression_pt: 0.0797 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3379 | Classification_binary: 2.8877 | Regression_cos_phi: 0.0467 | Regression_energy: 0.9095 | Regression_eta: 0.0504 | Regression_pt: 1.1088 | Regression_sin_phi: 0.0485 | LR: 2.25e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7599s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2075 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2886.6 | uss_MB: 1632.8 | child_rss_MB: 22913.8 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 16.3 +rank0 [2026-11-14 06:29:34,048] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 06:29:34,095] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 06:29:34,240] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 06:29:34,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:15,959] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 06:30:15,959] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:48,987] INFO : Step 69000/500000 rank0 | Train Loss: 5.6626 | Train Loss Unweighted: 3.0507 | Train Loss Components: Classification: 0.0226 | Classification_binary: 2.8122 | Regression_cos_phi: 0.0088 | Regression_energy: 0.0964 | Regression_eta: 0.0093 | Regression_pt: 0.0928 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2513 | Classification_binary: 2.8122 | Regression_cos_phi: 0.0969 | Regression_energy: 1.0447 | Regression_eta: 0.0732 | Regression_pt: 1.2913 | Regression_sin_phi: 0.0930 | LR: 2.30e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7634s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0024 | max_backward: 0.0268 | max_optimizer_step: 0.0858 | max_tensorboard_logging: 0.2151 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2886.8 | uss_MB: 1633.0 | child_rss_MB: 22914.3 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:30:58,369] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 06:30:58,369] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:31:40,059] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 06:31:40,059] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:04,282] INFO : Step 70000/500000 rank0 | Train Loss: 6.0207 | Train Loss Unweighted: 3.1402 | Train Loss Components: Classification: 0.0256 | Classification_binary: 2.8736 | Regression_cos_phi: 0.0145 | Regression_energy: 0.1053 | Regression_eta: 0.0130 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0145 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2841 | Classification_binary: 2.8736 | Regression_cos_phi: 0.1599 | Regression_energy: 1.1421 | Regression_eta: 0.1031 | Regression_pt: 1.3016 | Regression_sin_phi: 0.1563 | LR: 2.35e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.7762s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0242 | max_loss: 0.0024 | max_backward: 0.0271 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2237 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2883.2 | uss_MB: 1629.4 | child_rss_MB: 22915.4 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:32:04,284] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 06:32:04,331] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 06:32:04,332] INFO : Running validation on rank0 +rank0 [2026-11-14 06:32:04,332] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:32:04,332] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:32:04,438] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 06:32:04,523] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:32:04,602] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:32:04,653] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:32:04,699] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:32:04,745] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:32:04,791] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:32:04,839] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:32:04,904] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:32:04,982] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:32:05,060] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:32:05,107] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:32:05,153] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:32:05,198] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:32:05,243] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:32:05,290] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:32:05,361] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:32:05,366] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:32:05,368] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:32:05,369] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:32:05,371] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:32:05,372] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:32:05,374] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:32:05,376] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:32:05,377] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:32:05,379] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:32:05,380] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:32:05,382] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:32:05,383] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:32:05,385] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:32:05,386] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:32:05,388] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:32:06,373] INFO : VALIDATION | Step=70000/500000 | Train Loss=6.0207 | Valid Loss=9.5168 | Train Loss Unweighted=3.1402 | Valid Loss Unweighted=4.1249 | Stale=2 | ETA=540.2m +rank0 [2026-11-14 06:32:06,373] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:32:06,474] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:32:06,482] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:32:06,534] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:32:06,978] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:32:07,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:32:07,304] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:32:07,471] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:32:07,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:32:07,807] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:32:07,969] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:32:08,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:32:08,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:32:08,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:32:08,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:32:08,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:32:08,943] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:32:09,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:32:09,263] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:32:09,407] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:32:09,486] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:32:09,493] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:32:09,874] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:32:10,126] INFO : Plotted number of elements +rank0 [2026-11-14 06:32:11,060] INFO : Plotted jets +rank0 [2026-11-14 06:32:11,751] INFO : Plotted jet ratio +rank0 [2026-11-14 06:32:30,808] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 06:32:30,808] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:33:12,796] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 06:33:12,796] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:33:27,518] INFO : Step 71000/500000 rank0 | Train Loss: 5.1183 | Train Loss Unweighted: 2.8594 | Train Loss Components: Classification: 0.0174 | Classification_binary: 2.6540 | Regression_cos_phi: 0.0057 | Regression_energy: 0.0851 | Regression_eta: 0.0077 | Regression_pt: 0.0831 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.1929 | Classification_binary: 2.6540 | Regression_cos_phi: 0.0633 | Regression_energy: 0.9231 | Regression_eta: 0.0611 | Regression_pt: 1.1565 | Regression_sin_phi: 0.0674 | LR: 2.40e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7760s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0270 | max_optimizer_step: 0.0866 | max_tensorboard_logging: 0.2214 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3011.4 | uss_MB: 1757.7 | child_rss_MB: 22917.3 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:33:55,386] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 06:33:55,387] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:37,168] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 06:34:37,169] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:42,653] INFO : Step 72000/500000 rank0 | Train Loss: 5.0601 | Train Loss Unweighted: 2.8020 | Train Loss Components: Classification: 0.0268 | Classification_binary: 2.5958 | Regression_cos_phi: 0.0048 | Regression_energy: 0.0822 | Regression_eta: 0.0080 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0045 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2972 | Classification_binary: 2.5958 | Regression_cos_phi: 0.0529 | Regression_energy: 0.8908 | Regression_eta: 0.0635 | Regression_pt: 1.1111 | Regression_sin_phi: 0.0487 | LR: 2.45e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.7813s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0024 | max_backward: 0.0274 | max_optimizer_step: 0.0879 | max_tensorboard_logging: 0.2178 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3011.5 | uss_MB: 1757.7 | child_rss_MB: 22917.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 06:34:42,655] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 06:34:42,701] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 06:35:19,919] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 06:35:19,919] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:57,781] INFO : Step 73000/500000 rank0 | Train Loss: 5.5330 | Train Loss Unweighted: 2.9531 | Train Loss Components: Classification: 0.0271 | Classification_binary: 2.7190 | Regression_cos_phi: 0.0099 | Regression_energy: 0.0915 | Regression_eta: 0.0051 | Regression_pt: 0.0922 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.3012 | Classification_binary: 2.7190 | Regression_cos_phi: 0.1092 | Regression_energy: 0.9924 | Regression_eta: 0.0407 | Regression_pt: 1.2831 | Regression_sin_phi: 0.0874 | LR: 2.50e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.8127s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0244 | max_loss: 0.0023 | max_backward: 0.0275 | max_optimizer_step: 0.0868 | max_tensorboard_logging: 0.2149 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3011.5 | uss_MB: 1757.7 | child_rss_MB: 22918.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:36:02,224] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 06:36:02,224] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:43,914] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 06:36:43,914] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:12,579] INFO : Step 74000/500000 rank0 | Train Loss: 5.3633 | Train Loss Unweighted: 3.0301 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.8185 | Regression_cos_phi: 0.0065 | Regression_energy: 0.0860 | Regression_eta: 0.0068 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2156 | Classification_binary: 2.8185 | Regression_cos_phi: 0.0720 | Regression_energy: 0.9324 | Regression_eta: 0.0539 | Regression_pt: 1.1978 | Regression_sin_phi: 0.0732 | LR: 2.55e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7869s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0241 | max_loss: 0.0036 | max_backward: 0.0274 | max_optimizer_step: 0.0888 | max_tensorboard_logging: 0.2222 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2907.9 | uss_MB: 1654.2 | child_rss_MB: 22917.4 | allocated_MB: 126.1 | reserved_MB: 628.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 06:37:12,580] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 06:37:12,629] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 06:37:26,298] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 06:37:26,298] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:07,631] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 06:38:07,631] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:27,513] INFO : Step 75000/500000 rank0 | Train Loss: 4.9342 | Train Loss Unweighted: 2.7552 | Train Loss Components: Classification: 0.0204 | Classification_binary: 2.5549 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0809 | Regression_eta: 0.0099 | Regression_pt: 0.0754 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2266 | Classification_binary: 2.5549 | Regression_cos_phi: 0.0764 | Regression_energy: 0.8767 | Regression_eta: 0.0784 | Regression_pt: 1.0493 | Regression_sin_phi: 0.0718 | LR: 2.60e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.7732s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0240 | max_loss: 0.0023 | max_backward: 0.0262 | max_optimizer_step: 0.0861 | max_tensorboard_logging: 0.2205 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2911.2 | uss_MB: 1657.5 | child_rss_MB: 22918.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:38:49,798] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 06:38:49,798] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:39:30,070] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 06:39:30,070] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:39:40,358] INFO : Step 76000/500000 rank0 | Train Loss: 4.7154 | Train Loss Unweighted: 2.6986 | Train Loss Components: Classification: 0.0163 | Classification_binary: 2.5163 | Regression_cos_phi: 0.0082 | Regression_energy: 0.0709 | Regression_eta: 0.0051 | Regression_pt: 0.0757 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.1805 | Classification_binary: 2.5163 | Regression_cos_phi: 0.0907 | Regression_energy: 0.7690 | Regression_eta: 0.0406 | Regression_pt: 1.0528 | Regression_sin_phi: 0.0655 | LR: 2.65e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7697s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0241 | max_loss: 0.0023 | max_backward: 0.0273 | max_optimizer_step: 0.0862 | max_tensorboard_logging: 0.2176 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2911.3 | uss_MB: 1657.6 | child_rss_MB: 22918.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:39:40,360] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 06:39:40,407] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 06:40:12,484] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 06:40:12,484] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:53,966] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 06:40:53,966] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:55,140] INFO : Step 77000/500000 rank0 | Train Loss: 4.9293 | Train Loss Unweighted: 2.9131 | Train Loss Components: Classification: 0.0207 | Classification_binary: 2.7289 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0690 | Regression_eta: 0.0059 | Regression_pt: 0.0703 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2293 | Classification_binary: 2.7289 | Regression_cos_phi: 0.0663 | Regression_energy: 0.7481 | Regression_eta: 0.0465 | Regression_pt: 0.9784 | Regression_sin_phi: 0.1319 | LR: 2.70e-04 | DataLoad Time: 0.0003s | Model Forward Time: 0.7683s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0271 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0857 | max_tensorboard_logging: 0.2141 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2911.5 | uss_MB: 1657.8 | child_rss_MB: 22917.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:41:36,398] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 06:41:36,398] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:09,714] INFO : Step 78000/500000 rank0 | Train Loss: 4.6972 | Train Loss Unweighted: 2.7253 | Train Loss Components: Classification: 0.0258 | Classification_binary: 2.5448 | Regression_cos_phi: 0.0064 | Regression_energy: 0.0681 | Regression_eta: 0.0060 | Regression_pt: 0.0673 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2864 | Classification_binary: 2.5448 | Regression_cos_phi: 0.0710 | Regression_energy: 0.7379 | Regression_eta: 0.0471 | Regression_pt: 0.9356 | Regression_sin_phi: 0.0745 | LR: 2.75e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.7768s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0252 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0869 | max_tensorboard_logging: 0.2172 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2911.6 | uss_MB: 1657.8 | child_rss_MB: 22918.5 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:42:09,715] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 06:42:09,774] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 06:42:18,359] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 06:42:18,359] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:58,746] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 06:42:58,747] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:43:22,766] INFO : Step 79000/500000 rank0 | Train Loss: 4.5664 | Train Loss Unweighted: 2.5385 | Train Loss Components: Classification: 0.0218 | Classification_binary: 2.3515 | Regression_cos_phi: 0.0096 | Regression_energy: 0.0735 | Regression_eta: 0.0075 | Regression_pt: 0.0661 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2415 | Classification_binary: 2.3515 | Regression_cos_phi: 0.1066 | Regression_energy: 0.7973 | Regression_eta: 0.0592 | Regression_pt: 0.9200 | Regression_sin_phi: 0.0902 | LR: 2.80e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7643s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0243 | max_loss: 0.0023 | max_backward: 0.0271 | max_optimizer_step: 0.0867 | max_tensorboard_logging: 0.2167 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2911.3 | uss_MB: 1722.3 | child_rss_MB: 22918.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:43:40,721] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 06:43:40,721] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:44:22,378] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 06:44:22,379] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:44:38,014] INFO : Step 80000/500000 rank0 | Train Loss: 4.5300 | Train Loss Unweighted: 2.5459 | Train Loss Components: Classification: 0.0241 | Classification_binary: 2.3645 | Regression_cos_phi: 0.0071 | Regression_energy: 0.0683 | Regression_eta: 0.0050 | Regression_pt: 0.0675 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.0986 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0547 | Regression_energy: 10.8415 | Regression_eta: 7.9089 | Regression_pt: 13.9087 | Regression_sin_phi: 10.7507 | Task Weighted Losses: Classification: 0.2671 | Classification_binary: 2.3645 | Regression_cos_phi: 0.0785 | Regression_energy: 0.7403 | Regression_eta: 0.0398 | Regression_pt: 0.9385 | Regression_sin_phi: 0.1014 | LR: 2.85e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.7569s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0251 | max_loss: 0.0023 | max_backward: 0.0265 | max_optimizer_step: 0.0859 | max_tensorboard_logging: 0.2145 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2912.0 | uss_MB: 1723.0 | child_rss_MB: 22918.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:44:38,015] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 06:44:38,062] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 06:44:38,063] INFO : Running validation on rank0 +rank0 [2026-11-14 06:44:38,063] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:44:38,063] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:44:38,167] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 06:44:38,251] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:44:38,329] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:44:38,379] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:44:38,425] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:44:38,470] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:44:38,517] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:44:38,563] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:44:38,632] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:44:38,710] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:44:38,788] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:44:38,836] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:44:38,883] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:44:38,929] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:44:38,975] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:44:39,022] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:44:39,096] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:44:39,101] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:44:39,102] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:44:39,104] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:44:39,105] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:44:39,107] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:44:39,109] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:44:39,110] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:44:39,112] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:44:39,113] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:44:39,115] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:44:39,116] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:44:39,118] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:44:39,120] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:44:39,121] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:44:39,123] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:44:40,066] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.5300 | Valid Loss=10.2606 | Train Loss Unweighted=2.5459 | Valid Loss Unweighted=4.5677 | Stale=3 | ETA=527.6m +rank0 [2026-11-14 06:44:40,066] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:44:40,167] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:44:40,176] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:40,228] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:44:40,686] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:44:40,854] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:44:41,017] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:44:41,180] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:44:41,341] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:44:41,501] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:44:41,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:44:41,827] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:44:41,985] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:44:42,149] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:44:42,308] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:44:42,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:44:42,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:44:42,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:44:42,964] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:44:43,110] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:44:43,204] INFO : Time taken to make predictions on device 0 is: 0.05 min +rank0 [2026-11-14 06:44:43,212] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:43,600] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_12mm_seed12345_20261114_050401_337441/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:43,838] INFO : Plotted number of elements +rank0 [2026-11-14 06:44:44,791] INFO : Plotted jets +rank0 [2026-11-14 06:44:45,497] INFO : Plotted jet ratio +rank0 [2026-11-14 06:44:45,972] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 06:44:45,972] INFO : Training completed. Total time on device 0: 100.594min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..3b2d70e6ecbf984654a669df672162bcda74d6a2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d501d6f84cf5dacfe4e829b74860c0220668002eb0df37c516d06b9fc50ff8a0 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-80000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-80000.pth new file mode 100644 index 0000000000000000000000000000000000000000..a3399ba7b8602de811489c043185342a9d8cd44f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-80000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e94221a862865d0c6bbcb1141eb533dbd8d5a18987270df90818e36d50962fa2 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..68ba8cb1d8ccc65f6e27e49b63c17a519f0acafe --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 4.047906398773193, "Classification": 0.03491271659731865, "Regression_pt": 0.19679301977157593, "Regression_eta": 0.010869265533983707, "Regression_sin_phi": 0.00764417415484786, "Regression_cos_phi": 0.014010682702064514, "Regression_energy": 0.20095232129096985, "Total": 9.658344268798828, "Total_unweighted": 4.513088578823954}, "train_group": {}, "valid": {"Classification_binary": 3.9108524322509766, "Classification": 0.03643159940838814, "Regression_pt": 0.21664279699325562, "Regression_eta": 0.010460799559950829, "Regression_sin_phi": 0.011134270578622818, "Regression_cos_phi": 0.010292326100170612, "Regression_energy": 0.21312686800956726, "Total": 9.937812805175781, "Total_unweighted": 4.408941092900932, "metrics/particle/class_1/efficiency": 0.886261522769928, "metrics/particle/class_1/pid_accuracy": 0.9831691980361938, "metrics/particle/class_2/efficiency": 0.1854695826768875, "metrics/particle/class_2/pid_accuracy": 0.2675159275531769, "metrics/particle/class_3/efficiency": 0.6814606189727783, "metrics/particle/class_3/pid_accuracy": 0.9747755527496338, "metrics/particle/class_4/efficiency": 0.8510638475418091, "metrics/particle/class_4/pid_accuracy": 0.0357142873108387, "metrics/particle/class_5/efficiency": 0.9611111283302307, "metrics/particle/class_5/pid_accuracy": 0.13294798135757446, "metrics/particle/count/bias_mean": -1.8960000276565552, "metrics/particle/count/mae": 5.97599983215332, "metrics/particle/count/prediction_mean": 95.2959976196289, "metrics/particle/count/target_mean": 97.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.0847148448228836, "metrics/particle/event/energy_response_mean": 0.937248706817627, "metrics/particle/event/met_abs_error": 13.20606803894043, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08545704931020737, "metrics/particle/event/scalar_pt_response_mean": 0.9399251937866211, "metrics/particle/event/vector_pt_closure": 0.09018290042877197, "metrics/particle/matched/delta_eta_abs_mean": 0.02135424315929413, "metrics/particle/matched/delta_phi_abs_mean": 0.01936238259077072, "metrics/particle/matched/delta_r_mean": 0.03206022083759308, "metrics/particle/matched/energy_relative_abs_mean": 0.09153047204017639, "metrics/particle/matched/pid_accuracy": 0.9450139999389648, "metrics/particle/matched/pt_relative_abs_mean": 0.09002381563186646, "metrics/particle/matching/duplicate_fraction": 0.03786098212003708, "metrics/particle/matching/efficiency": 0.750720202922821, "metrics/particle/matching/f1": 0.7581148147583008, "metrics/particle/matching/purity": 0.7656564712524414, "metrics/particle/matching/target_coverage_dr0p05": 0.6270063519477844, "metrics/particle/matching/target_coverage_dr0p10": 0.7867314219474792, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6725245118141174, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8110132813453674}, "step_train_time": 0.5033819675445557, "step_valid_time": 2.827445324975997, "step_total_time": 1125.9072289466858, "elapsed_training_loop_seconds": 1125.9072289466858, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.96722163061684, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20520508074827026, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1540, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1334, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1270, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8679245283018868, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8662337662337662, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.867078323041924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13376623376623376, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8262849707221861, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8246753246753247, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8254793630159246, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8679245283018868, "step_test_time": 5.112317278981209, "step_plot_time": 3.7475358429946937, "step_validation_cycle_time": 12.185602041019592} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..0ce76537ba2faf09376d5fb26d5822f9578b8c6b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 3.755035161972046, "Classification": 0.030766112729907036, "Regression_pt": 0.18972326815128326, "Regression_eta": 0.008292192593216896, "Regression_sin_phi": 0.010117613710463047, "Regression_cos_phi": 0.010780028067529202, "Regression_energy": 0.18777777254581451, "Total": 9.049174308776855, "Total_unweighted": 4.19249214977026}, "train_group": {}, "valid": {"Classification_binary": 3.758218288421631, "Classification": 0.034800175577402115, "Regression_pt": 0.20862939953804016, "Regression_eta": 0.009746267460286617, "Regression_sin_phi": 0.01023948099464178, "Regression_cos_phi": 0.00960924569517374, "Regression_energy": 0.20414426922798157, "Total": 9.536052703857422, "Total_unweighted": 4.235387126915157, "metrics/particle/class_1/efficiency": 0.9002457857131958, "metrics/particle/class_1/pid_accuracy": 0.981924295425415, "metrics/particle/class_2/efficiency": 0.2244536280632019, "metrics/particle/class_2/pid_accuracy": 0.38684210181236267, "metrics/particle/class_3/efficiency": 0.7089443802833557, "metrics/particle/class_3/pid_accuracy": 0.9787670969963074, "metrics/particle/class_4/efficiency": 0.8875380158424377, "metrics/particle/class_4/pid_accuracy": 0.051369864493608475, "metrics/particle/class_5/efficiency": 0.9611111283302307, "metrics/particle/class_5/pid_accuracy": 0.22543352842330933, "metrics/particle/count/bias_mean": -0.335999995470047, "metrics/particle/count/mae": 5.383999824523926, "metrics/particle/count/prediction_mean": 96.85600280761719, "metrics/particle/count/target_mean": 97.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.06883686035871506, "metrics/particle/event/energy_response_mean": 0.9625418782234192, "metrics/particle/event/met_abs_error": 11.73847484588623, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07211460918188095, "metrics/particle/event/scalar_pt_response_mean": 0.9654827117919922, "metrics/particle/event/vector_pt_closure": 0.08092375099658966, "metrics/particle/matched/delta_eta_abs_mean": 0.020168418064713478, "metrics/particle/matched/delta_phi_abs_mean": 0.017607519403100014, "metrics/particle/matched/delta_r_mean": 0.029780156910419464, "metrics/particle/matched/energy_relative_abs_mean": 0.09282054752111435, "metrics/particle/matched/pid_accuracy": 0.947194516658783, "metrics/particle/matched/pt_relative_abs_mean": 0.0889245942234993, "metrics/particle/matching/duplicate_fraction": 0.03985297679901123, "metrics/particle/matching/efficiency": 0.7723680734634399, "metrics/particle/matching/f1": 0.7737054824829102, "metrics/particle/matching/purity": 0.7750474810600281, "metrics/particle/matching/target_coverage_dr0p05": 0.6644991636276245, "metrics/particle/matching/target_coverage_dr0p10": 0.8087085485458374, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6825664639472961, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.822536826133728}, "step_train_time": 0.5048179626464844, "step_valid_time": 3.1001110979705118, "step_total_time": 2254.600822210312, "elapsed_training_loop_seconds": 2254.600822210312, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9788676780815435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19157644981597066, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1531, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1362, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1313, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8861418347430059, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8896146309601568, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8878748370273793, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11038536903984324, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8542615484710475, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8576094056172436, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8559322033898306, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8861418347430059, "step_test_time": 6.152728537039366, "step_plot_time": 3.3736198699916713, "step_validation_cycle_time": 13.22875212301733} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..c0bb2157ea489c8198535b495d20fdbaba7cda52 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 3.3508493900299072, "Classification": 0.03731197491288185, "Regression_pt": 0.16957814991474152, "Regression_eta": 0.004778955597430468, "Regression_sin_phi": 0.00526938121765852, "Regression_cos_phi": 0.006440068129450083, "Regression_energy": 0.16827520728111267, "Total": 8.10025405883789, "Total_unweighted": 3.7425031270831823}, "train_group": {}, "valid": {"Classification_binary": 3.6424074172973633, "Classification": 0.03376489132642746, "Regression_pt": 0.20280314981937408, "Regression_eta": 0.009351449087262154, "Regression_sin_phi": 0.010002302937209606, "Regression_cos_phi": 0.0093085253611207, "Regression_energy": 0.19866140186786652, "Total": 9.259737014770508, "Total_unweighted": 4.106299137696624, "metrics/particle/class_1/efficiency": 0.898381233215332, "metrics/particle/class_1/pid_accuracy": 0.9811320900917053, "metrics/particle/class_2/efficiency": 0.26461902260780334, "metrics/particle/class_2/pid_accuracy": 0.4508928656578064, "metrics/particle/class_3/efficiency": 0.7319607734680176, "metrics/particle/class_3/pid_accuracy": 0.9767811894416809, "metrics/particle/class_4/efficiency": 0.8571428656578064, "metrics/particle/class_4/pid_accuracy": 0.10283687710762024, "metrics/particle/class_5/efficiency": 0.9555555582046509, "metrics/particle/class_5/pid_accuracy": 0.2616279125213623, "metrics/particle/count/bias_mean": 1.2519999742507935, "metrics/particle/count/mae": 5.228000164031982, "metrics/particle/count/prediction_mean": 98.44400024414062, "metrics/particle/count/target_mean": 97.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.06115042790770531, "metrics/particle/event/energy_response_mean": 0.9977719783782959, "metrics/particle/event/met_abs_error": 10.875683784484863, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06162707880139351, "metrics/particle/event/scalar_pt_response_mean": 1.000388741493225, "metrics/particle/event/vector_pt_closure": 0.0704115703701973, "metrics/particle/matched/delta_eta_abs_mean": 0.018319426104426384, "metrics/particle/matched/delta_phi_abs_mean": 0.01871260441839695, "metrics/particle/matched/delta_r_mean": 0.029089869931340218, "metrics/particle/matched/energy_relative_abs_mean": 0.0978604331612587, "metrics/particle/matched/pid_accuracy": 0.9474236965179443, "metrics/particle/matched/pt_relative_abs_mean": 0.08857803046703339, "metrics/particle/matching/duplicate_fraction": 0.040347810834646225, "metrics/particle/matching/efficiency": 0.783562421798706, "metrics/particle/matching/f1": 0.7785479426383972, "metrics/particle/matching/purity": 0.7735971808433533, "metrics/particle/matching/target_coverage_dr0p05": 0.677874743938446, "metrics/particle/matching/target_coverage_dr0p10": 0.8181331753730774, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6955716609954834, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8372705578804016}, "step_train_time": 0.5206878185272217, "step_valid_time": 3.045110734004993, "step_total_time": 3390.208925962448, "elapsed_training_loop_seconds": 3390.208925962448, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9969863234183436, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16678877470757747, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1539, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1370, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1339, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8913467794404685, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8901884340480831, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8907672301690507, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10981156595191682, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8711776187378009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8700454840805718, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8706111833550065, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8913467794404685, "step_test_time": 4.577003402984701, "step_plot_time": 3.3257214829791337, "step_validation_cycle_time": 11.520161048974842} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..9c391fd5572fd50f59a7be58e1f4857ca6b7cbf7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 3.5788309574127197, "Classification": 0.03496738523244858, "Regression_pt": 0.16577251255512238, "Regression_eta": 0.012889145873486996, "Regression_sin_phi": 0.01443580910563469, "Regression_cos_phi": 0.014162941835820675, "Regression_energy": 0.16210047900676727, "Total": 8.430500984191895, "Total_unweighted": 3.9831592310220003}, "train_group": {}, "valid": {"Classification_binary": 3.4741718769073486, "Classification": 0.030074467882514, "Regression_pt": 0.19464339315891266, "Regression_eta": 0.009383060038089752, "Regression_sin_phi": 0.009678435511887074, "Regression_cos_phi": 0.009100694209337234, "Regression_energy": 0.19111716747283936, "Total": 8.850305557250977, "Total_unweighted": 3.9181690951809287, "metrics/particle/class_1/efficiency": 0.9070259928703308, "metrics/particle/class_1/pid_accuracy": 0.9799103140830994, "metrics/particle/class_2/efficiency": 0.2593030035495758, "metrics/particle/class_2/pid_accuracy": 0.42596811056137085, "metrics/particle/class_3/efficiency": 0.7265222668647766, "metrics/particle/class_3/pid_accuracy": 0.9794145226478577, "metrics/particle/class_4/efficiency": 0.8723404407501221, "metrics/particle/class_4/pid_accuracy": 0.18118466436862946, "metrics/particle/class_5/efficiency": 0.9555555582046509, "metrics/particle/class_5/pid_accuracy": 0.5, "metrics/particle/count/bias_mean": 2.671999931335449, "metrics/particle/count/mae": 5.383999824523926, "metrics/particle/count/prediction_mean": 99.86399841308594, "metrics/particle/count/target_mean": 97.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.05773709714412689, "metrics/particle/event/energy_response_mean": 1.0084294080734253, "metrics/particle/event/met_abs_error": 10.320469856262207, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05617302283644676, "metrics/particle/event/scalar_pt_response_mean": 1.0129504203796387, "metrics/particle/event/vector_pt_closure": 0.06545049697160721, "metrics/particle/matched/delta_eta_abs_mean": 0.020995933562517166, "metrics/particle/matched/delta_phi_abs_mean": 0.017463000491261482, "metrics/particle/matched/delta_r_mean": 0.030356086790561676, "metrics/particle/matched/energy_relative_abs_mean": 0.09346853941679001, "metrics/particle/matched/pid_accuracy": 0.9506314992904663, "metrics/particle/matched/pt_relative_abs_mean": 0.08863113075494766, "metrics/particle/matching/duplicate_fraction": 0.04077545553445816, "metrics/particle/matching/efficiency": 0.7852909564971924, "metrics/particle/matching/f1": 0.7746427655220032, "metrics/particle/matching/purity": 0.7642794251441956, "metrics/particle/matching/target_coverage_dr0p05": 0.6576672792434692, "metrics/particle/matching/target_coverage_dr0p10": 0.8203967213630676, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6997695565223694, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8439789414405823}, "step_train_time": 0.5422346591949463, "step_valid_time": 3.0096948530408554, "step_total_time": 4519.461629867554, "elapsed_training_loop_seconds": 4519.461629867554, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003638939675446, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1539782797216238, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1529, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1375, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1349, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8945998698763825, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8992805755395683, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8969341161121983, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10071942446043165, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8776837996096292, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.882275997383911, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8799739073711677, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8945998698763825, "step_test_time": 4.526663433003705, "step_plot_time": 3.3197973169735633, "step_validation_cycle_time": 11.405437969020568} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..6e383cd1964e1ff2b79656123bc500e0da2545f8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 3.34846830368042, "Classification": 0.024878952652215958, "Regression_pt": 0.15517234802246094, "Regression_eta": 0.0120466947555542, "Regression_sin_phi": 0.01177880261093378, "Regression_cos_phi": 0.01294709648936987, "Regression_energy": 0.15030261874198914, "Total": 7.7647504806518555, "Total_unweighted": 3.715594816952944}, "train_group": {}, "valid": {"Classification_binary": 3.398388147354126, "Classification": 0.028745677322149277, "Regression_pt": 0.19652824103832245, "Regression_eta": 0.009002959355711937, "Regression_sin_phi": 0.009297116659581661, "Regression_cos_phi": 0.008729079738259315, "Regression_energy": 0.19459721446037292, "Total": 8.81229019165039, "Total_unweighted": 3.8452884359285235, "metrics/particle/class_1/efficiency": 0.911263644695282, "metrics/particle/class_1/pid_accuracy": 0.9807477593421936, "metrics/particle/class_2/efficiency": 0.24867099523544312, "metrics/particle/class_2/pid_accuracy": 0.4679335057735443, "metrics/particle/class_3/efficiency": 0.7332232594490051, "metrics/particle/class_3/pid_accuracy": 0.9790728688240051, "metrics/particle/class_4/efficiency": 0.8723404407501221, "metrics/particle/class_4/pid_accuracy": 0.18815331161022186, "metrics/particle/class_5/efficiency": 0.9611111283302307, "metrics/particle/class_5/pid_accuracy": 0.5028901696205139, "metrics/particle/count/bias_mean": 1.0080000162124634, "metrics/particle/count/mae": 4.736000061035156, "metrics/particle/count/prediction_mean": 98.19999694824219, "metrics/particle/count/target_mean": 97.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.060431286692619324, "metrics/particle/event/energy_response_mean": 0.9673931002616882, "metrics/particle/event/met_abs_error": 9.339198112487793, "metrics/particle/event/scalar_pt_relative_abs_error": 0.056078288704156876, "metrics/particle/event/scalar_pt_response_mean": 0.9696944952011108, "metrics/particle/event/vector_pt_closure": 0.06336898356676102, "metrics/particle/matched/delta_eta_abs_mean": 0.019394321367144585, "metrics/particle/matched/delta_phi_abs_mean": 0.01735205575823784, "metrics/particle/matched/delta_r_mean": 0.02899051643908024, "metrics/particle/matched/energy_relative_abs_mean": 0.09371477365493774, "metrics/particle/matched/pid_accuracy": 0.9526664018630981, "metrics/particle/matched/pt_relative_abs_mean": 0.08995641022920609, "metrics/particle/matching/duplicate_fraction": 0.03649694472551346, "metrics/particle/matching/efficiency": 0.7894888520240784, "metrics/particle/matching/f1": 0.7854160070419312, "metrics/particle/matching/purity": 0.7813849449157715, "metrics/particle/matching/target_coverage_dr0p05": 0.6819491386413574, "metrics/particle/matching/target_coverage_dr0p10": 0.8223310708999634, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7002222537994385, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8437319993972778}, "step_train_time": 0.5212991237640381, "step_valid_time": 2.953678696008865, "step_total_time": 5652.9423933029175, "elapsed_training_loop_seconds": 5652.9423933029175, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.988628068250184, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16443158893589493, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1514, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1379, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1355, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8972023422251139, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9108322324966975, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9039659128154703, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0891677675033025, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8815875081327261, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8949801849405549, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8882333661094723, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8972023422251139, "step_test_time": 4.737001038971357, "step_plot_time": 3.2326010090182535, "step_validation_cycle_time": 11.467040440998971} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_60000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_60000.json new file mode 100644 index 0000000000000000000000000000000000000000..d53ab9566beb340329941bbb1de6211e3d7a8eac --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_60000.json @@ -0,0 +1 @@ +{"epoch": 106.76156583629893, "train": {"Classification_binary": 2.9626874923706055, "Classification": 0.027811478823423386, "Regression_pt": 0.11601206660270691, "Regression_eta": 0.009595781564712524, "Regression_sin_phi": 0.0077623105607926846, "Regression_cos_phi": 0.006769426167011261, "Regression_energy": 0.11287444829940796, "Total": 6.333952903747559, "Total_unweighted": 3.24351300438866}, "train_group": {}, "valid": {"Classification_binary": 3.389996290206909, "Classification": 0.028687071055173874, "Regression_pt": 0.20304207503795624, "Regression_eta": 0.00884409248828888, "Regression_sin_phi": 0.009012368507683277, "Regression_cos_phi": 0.00842109601944685, "Regression_energy": 0.20039425790309906, "Total": 8.9484224319458, "Total_unweighted": 3.8483972512185574, "metrics/particle/class_1/efficiency": 0.9087210893630981, "metrics/particle/class_1/pid_accuracy": 0.9800410270690918, "metrics/particle/class_2/efficiency": 0.26934435963630676, "metrics/particle/class_2/pid_accuracy": 0.5416666865348816, "metrics/particle/class_3/efficiency": 0.7453626990318298, "metrics/particle/class_3/pid_accuracy": 0.9771987199783325, "metrics/particle/class_4/efficiency": 0.8662614226341248, "metrics/particle/class_4/pid_accuracy": 0.1894736886024475, "metrics/particle/class_5/efficiency": 0.9611111283302307, "metrics/particle/class_5/pid_accuracy": 0.5375722646713257, "metrics/particle/count/bias_mean": 1.6039999723434448, "metrics/particle/count/mae": 5.156000137329102, "metrics/particle/count/prediction_mean": 98.7959976196289, "metrics/particle/count/target_mean": 97.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.05748842656612396, "metrics/particle/event/energy_response_mean": 0.9963624477386475, "metrics/particle/event/met_abs_error": 9.110469818115234, "metrics/particle/event/scalar_pt_relative_abs_error": 0.055367324501276016, "metrics/particle/event/scalar_pt_response_mean": 0.9993220567703247, "metrics/particle/event/vector_pt_closure": 0.06031731143593788, "metrics/particle/matched/delta_eta_abs_mean": 0.01938633993268013, "metrics/particle/matched/delta_phi_abs_mean": 0.017949974164366722, "metrics/particle/matched/delta_r_mean": 0.029445819556713104, "metrics/particle/matched/energy_relative_abs_mean": 0.09658314287662506, "metrics/particle/matched/pid_accuracy": 0.952928364276886, "metrics/particle/matched/pt_relative_abs_mean": 0.08881019055843353, "metrics/particle/matching/duplicate_fraction": 0.03793675824999809, "metrics/particle/matching/efficiency": 0.7947567701339722, "metrics/particle/matching/f1": 0.7882523536682129, "metrics/particle/matching/purity": 0.7818534970283508, "metrics/particle/matching/target_coverage_dr0p05": 0.6738002896308899, "metrics/particle/matching/target_coverage_dr0p10": 0.825376570224762, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7104700207710266, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8499053716659546}, "step_train_time": 0.543316125869751, "step_valid_time": 3.0415816939785145, "step_total_time": 6788.611540794373, "elapsed_training_loop_seconds": 6788.611540794373, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9995522460320504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15006191538943825, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1523, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1384, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1356, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.900455432661028, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9087327642810243, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9045751633986928, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0912672357189757, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8822381262199089, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8903479973736047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8862745098039215, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.900455432661028, "step_test_time": 4.911544725007843, "step_plot_time": 3.3138791330275126, "step_validation_cycle_time": 11.707692619005684} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_70000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_70000.json new file mode 100644 index 0000000000000000000000000000000000000000..c1f90e1d748c31b00c5e62d2392dffdfb9dd91d8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_70000.json @@ -0,0 +1 @@ +{"epoch": 124.55516014234875, "train": {"Classification_binary": 2.7871880531311035, "Classification": 0.02288082242012024, "Regression_pt": 0.09837143123149872, "Regression_eta": 0.007841509766876698, "Regression_sin_phi": 0.00896299909800291, "Regression_cos_phi": 0.00998236145824194, "Regression_energy": 0.09799132496118546, "Total": 5.732627868652344, "Total_unweighted": 3.0332185020670295}, "train_group": {}, "valid": {"Classification_binary": 3.5038537979125977, "Classification": 0.029884055256843567, "Regression_pt": 0.2220441699028015, "Regression_eta": 0.008846208453178406, "Regression_sin_phi": 0.009006254374980927, "Regression_cos_phi": 0.008350864052772522, "Regression_energy": 0.218821719288826, "Total": 9.537223815917969, "Total_unweighted": 4.000807069242001, "metrics/particle/class_1/efficiency": 0.9126197099685669, "metrics/particle/class_1/pid_accuracy": 0.9798476696014404, "metrics/particle/class_2/efficiency": 0.2640283405780792, "metrics/particle/class_2/pid_accuracy": 0.5436241626739502, "metrics/particle/class_3/efficiency": 0.7262309193611145, "metrics/particle/class_3/pid_accuracy": 0.9741909503936768, "metrics/particle/class_4/efficiency": 0.8449848294258118, "metrics/particle/class_4/pid_accuracy": 0.23021583259105682, "metrics/particle/class_5/efficiency": 0.9611111283302307, "metrics/particle/class_5/pid_accuracy": 0.5780346989631653, "metrics/particle/count/bias_mean": 0.19599999487400055, "metrics/particle/count/mae": 5.228000164031982, "metrics/particle/count/prediction_mean": 97.38800048828125, "metrics/particle/count/target_mean": 97.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.06252363324165344, "metrics/particle/event/energy_response_mean": 0.965347945690155, "metrics/particle/event/met_abs_error": 8.446518898010254, "metrics/particle/event/scalar_pt_relative_abs_error": 0.060470763593912125, "metrics/particle/event/scalar_pt_response_mean": 0.9693287014961243, "metrics/particle/event/vector_pt_closure": 0.05981817469000816, "metrics/particle/matched/delta_eta_abs_mean": 0.019232187420129776, "metrics/particle/matched/delta_phi_abs_mean": 0.01804475672543049, "metrics/particle/matched/delta_r_mean": 0.02941523678600788, "metrics/particle/matched/energy_relative_abs_mean": 0.09374495595693588, "metrics/particle/matched/pid_accuracy": 0.9529356360435486, "metrics/particle/matched/pt_relative_abs_mean": 0.08457843959331512, "metrics/particle/matching/duplicate_fraction": 0.03643159195780754, "metrics/particle/matching/efficiency": 0.7878837585449219, "metrics/particle/matching/f1": 0.7870901226997375, "metrics/particle/matching/purity": 0.7862980961799622, "metrics/particle/matching/target_coverage_dr0p05": 0.6603423953056335, "metrics/particle/matching/target_coverage_dr0p10": 0.8167338967323303, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7093176245689392, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8429500460624695}, "step_train_time": 0.5342497825622559, "step_valid_time": 3.0343004589667544, "step_total_time": 7924.2665338516235, "elapsed_training_loop_seconds": 7924.2665338516235, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9799559588226247, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.151735692832065, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1518, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1377, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1358, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8959011060507482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9071146245059288, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9014729950900163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09288537549407115, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8835393623942746, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8945981554677207, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8890343698854337, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8959011060507482, "step_test_time": 4.7910252379952, "step_plot_time": 3.334346054994967, "step_validation_cycle_time": 11.63585800700821} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_80000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_80000.json new file mode 100644 index 0000000000000000000000000000000000000000..1a158ef696b3c88d7817e9ac96488e8e80ae281e --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/history/step_80000.json @@ -0,0 +1 @@ +{"epoch": 142.34875444839858, "train": {"Classification_binary": 2.2896721363067627, "Classification": 0.021268203854560852, "Regression_pt": 0.059610556811094284, "Regression_eta": 0.005329466890543699, "Regression_sin_phi": 0.00543194031342864, "Regression_cos_phi": 0.008943123742938042, "Regression_energy": 0.06184519827365875, "Total": 4.220189094543457, "Total_unweighted": 2.452100626192987}, "train_group": {}, "valid": {"Classification_binary": 3.864603281021118, "Classification": 0.031170925125479698, "Regression_pt": 0.24233151972293854, "Regression_eta": 0.008463987149298191, "Regression_sin_phi": 0.009011328220367432, "Regression_cos_phi": 0.008363064378499985, "Regression_energy": 0.24049578607082367, "Total": 10.424847602844238, "Total_unweighted": 4.404439891688526, "metrics/particle/class_1/efficiency": 0.9034664034843445, "metrics/particle/class_1/pid_accuracy": 0.9771106839179993, "metrics/particle/class_2/efficiency": 0.28824570775032043, "metrics/particle/class_2/pid_accuracy": 0.5491803288459778, "metrics/particle/class_3/efficiency": 0.7133145332336426, "metrics/particle/class_3/pid_accuracy": 0.9726344347000122, "metrics/particle/class_4/efficiency": 0.8449848294258118, "metrics/particle/class_4/pid_accuracy": 0.2374100685119629, "metrics/particle/class_5/efficiency": 0.9611111283302307, "metrics/particle/class_5/pid_accuracy": 0.5086705088615417, "metrics/particle/count/bias_mean": 0.19599999487400055, "metrics/particle/count/mae": 5.6519999504089355, "metrics/particle/count/prediction_mean": 97.38800048828125, "metrics/particle/count/target_mean": 97.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.05911681056022644, "metrics/particle/event/energy_response_mean": 0.9935271143913269, "metrics/particle/event/met_abs_error": 9.56840705871582, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05700228735804558, "metrics/particle/event/scalar_pt_response_mean": 0.9956931471824646, "metrics/particle/event/vector_pt_closure": 0.06280514597892761, "metrics/particle/matched/delta_eta_abs_mean": 0.018261604011058807, "metrics/particle/matched/delta_phi_abs_mean": 0.018629638478159904, "metrics/particle/matched/delta_r_mean": 0.029120605438947678, "metrics/particle/matched/energy_relative_abs_mean": 0.09257248044013977, "metrics/particle/matched/pid_accuracy": 0.94921875, "metrics/particle/matched/pt_relative_abs_mean": 0.08627600222826004, "metrics/particle/matching/duplicate_fraction": 0.039224546402692795, "metrics/particle/matching/efficiency": 0.7796526551246643, "metrics/particle/matching/f1": 0.7788673043251038, "metrics/particle/matching/purity": 0.7780835628509521, "metrics/particle/matching/target_coverage_dr0p05": 0.6575026512145996, "metrics/particle/matching/target_coverage_dr0p10": 0.8090789318084717, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6981232762336731, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8346777558326721}, "step_train_time": 0.5213687419891357, "step_valid_time": 3.0519486449775286, "step_total_time": 9061.787279129028, "elapsed_training_loop_seconds": 9061.787279129028, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9988941515355578, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1557990572660375, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1523, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1375, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1353, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8945998698763825, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9028233749179252, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8986928104575164, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09717662508207485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8802862719583604, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8883782009192384, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.884313725490196, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8945998698763825, "step_test_time": 4.915793558000587, "step_plot_time": 3.3285685760201886, "step_validation_cycle_time": 11.758462058962323} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..f713a4f0d979c965ddcfb3a80f85696c47c47f46 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_15mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1bdf85c2e164f33659c6fc309d3f844428ee3d46 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0d6a3bc67f8dbc26f9b1246c27bf2e3b7721f94dddf15cc6b154c87dea6457b +size 4394 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ef3c49b9c46c574ae99991c3d8bba9b21df0cc9b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..3a3d5336f22e0b306cb3f02b64ad8a2519cdca66 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..52522e68c7f73245e5d17581afb5301bab56ef14 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..94e38a832233b32e8735e03db71a7a304afb0ecc Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a72f1615821d5b8545f04d6d56fd09733a1977f5 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..a3ee48bc89adb84925aba263769273a002a075ce Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a2bbfbceb83e0936245191dcde98cbc95d1faadc Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..a6105fdb2887b667eb8ff7d16d02c12c723c8027 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1571945af092e4643a49cd8646939c0b7764a17e Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..f246d8b218b58da6646268eaca558ab954fe2545 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a6067c809311bf9c96b1150c1f7fc0735bd8dc30 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..b8fb220c80b65769b059459726da4b65872449e1 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/plots_step_80000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/runs/train/events.out.tfevents.1794652427.workergpu154.2708352.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/runs/train/events.out.tfevents.1794652427.workergpu154.2708352.0 new file mode 100644 index 0000000000000000000000000000000000000000..a6a039752b43ef701e11dce637e7b0baf421dd20 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/runs/train/events.out.tfevents.1794652427.workergpu154.2708352.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b514f82d89cc13113614533d7e3995aadd1fac014531aa1f1b3732332ba96c1d +size 2953648 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/runs/valid/events.out.tfevents.1794652427.workergpu154.2708352.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/runs/valid/events.out.tfevents.1794652427.workergpu154.2708352.1 new file mode 100644 index 0000000000000000000000000000000000000000..e3b1c9a8fdac18c13f769fbb57a86d78a236d51d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/runs/valid/events.out.tfevents.1794652427.workergpu154.2708352.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:716e6f3833db7a569fa87065764b1f4c4fc61eb2777b79ce01b0b1f2ddc826ac +size 4007665 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..45b509cf1f01de6ce4d4dec5a859212c76464716 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_hcal_15mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_15mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_15mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_15mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3368e32ba9b02283c1572d1ff0df1c953ea595c7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_15mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/train.log new file mode 100644 index 0000000000000000000000000000000000000000..80ed5fe4305b4d165f6ac96cfc426fceba684bfd --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/train.log @@ -0,0 +1,1353 @@ +rank0 [2026-11-14 05:33:44,825] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-14 05:33:44,826] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 05:33:45,302] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 05:33:45,303] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 05:33:45,304] INFO : Instantiating model +rank0 [2026-11-14 05:33:45,327] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 05:33:45,328] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 05:33:45,329] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 05:33:45,329] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 05:33:45,329] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 05:33:45,330] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 05:33:45,330] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,332] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 05:33:45,332] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,335] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 05:33:45,335] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,338] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 05:33:45,338] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,341] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 05:33:45,341] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,344] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 05:33:45,344] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,346] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 05:33:45,347] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 05:33:45,347] INFO : Initializing output DNNs +rank0 [2026-11-14 05:33:45,347] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,350] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,357] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 05:33:45,358] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 05:33:45,358] INFO : jagged_attention enabled=True +rank0 [2026-11-14 05:33:45,358] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 05:33:45,358] INFO : task_queries enabled=True +rank0 [2026-11-14 05:33:45,358] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 05:33:45,358] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 05:33:45,358] INFO : classification_query parameters: 256 +rank0 [2026-11-14 05:33:45,358] INFO : regression_query parameters: 256 +rank0 [2026-11-14 05:33:45,358] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 05:33:45,358] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 05:33:45,358] INFO : output_mode=elementwise +rank0 [2026-11-14 05:33:45,358] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 05:33:45,358] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 05:33:45,358] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 05:33:45,358] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 05:33:45,358] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 05:33:45,358] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 05:33:45,358] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 05:33:45,358] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 05:33:45,359] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 05:33:45,360] INFO : MLPF __init__ done +rank0 [2026-11-14 05:33:45,360] INFO : Instantiated model +rank0 [2026-11-14 05:33:45,360] INFO : Moving model to device rank=0 +rank0 [2026-11-14 05:33:45,372] INFO : Moved model to device rank=0 +rank0 [2026-11-14 05:33:45,373] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 05:33:45,471] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 05:33:45,471] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 05:33:45,472] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 05:33:45,472] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 05:33:45,472] INFO : Total parameters: 3910930 +rank0 [2026-11-14 05:33:45,475] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 05:33:45,479] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794 +rank0 [2026-11-14 05:33:45,479] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794 +rank0 [2026-11-14 05:33:45,479] INFO : Getting dataloaders +rank0 [2026-11-14 05:33:45,479] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:33:46,703] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,756] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,819] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,869] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,897] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,936] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,982] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:47,038] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:47,081] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:47,122] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:47,122] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 05:33:47,122] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 05:33:47,122] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 05:33:47,122] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:33:47,131] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,140] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,148] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,158] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,166] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,173] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,181] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,188] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,197] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,206] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,206] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 05:33:47,207] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 05:33:47,250] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:33:47,399] INFO : Got dataloaders +rank0 [2026-11-14 05:33:47,399] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 05:33:47,399] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 05:33:47,400] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 05:33:47,406] INFO : Creating train_iterator +rank0 [2026-11-14 05:33:47,406] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:33:47,433] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:33:47,588] INFO : Created train_iterator +rank0 [2026-11-14 05:34:00,873] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.12873676776886, 'Classification': 0.6399854928255081, 'Regression_pt': 0.5156782293319702, 'Regression_eta': 0.8953252410888672, 'Regression_sin_phi': 0.6647196558117866, 'Regression_cos_phi': 0.6473717069625855, 'Regression_energy': 0.657646809220314}, weights={'Classification_binary': 1.0, 'Classification': 11.138903617858887, 'Regression_pt': 13.824002265930176, 'Regression_eta': 7.962175369262695, 'Regression_sin_phi': 10.72442626953125, 'Regression_cos_phi': 11.01181411743164, 'Regression_energy': 10.839764595031738} +rank0 [2026-11-14 05:34:53,616] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 05:34:53,616] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:43,505] INFO : Step 1000/500000 rank0 | Train Loss: 11.6596 | Train Loss Unweighted: 5.1031 | Train Loss Components: Classification: 0.0671 | Classification_binary: 4.5029 | Regression_cos_phi: 0.0193 | Regression_energy: 0.2242 | Regression_eta: 0.0294 | Regression_pt: 0.2394 | Regression_sin_phi: 0.0208 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.7477 | Classification_binary: 4.5029 | Regression_cos_phi: 0.2126 | Regression_energy: 2.4305 | Regression_eta: 0.2339 | Regression_pt: 3.3089 | Regression_sin_phi: 0.2231 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5817s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0388 | max_loss: 0.0029 | max_backward: 0.0491 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1917 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2578.7 | uss_MB: 1412.7 | child_rss_MB: 22594.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:35:57,450] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 05:35:57,450] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:00,522] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 05:37:00,522] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:36,092] INFO : Step 2000/500000 rank0 | Train Loss: 10.5215 | Train Loss Unweighted: 4.7396 | Train Loss Components: Classification: 0.0483 | Classification_binary: 4.2137 | Regression_cos_phi: 0.0150 | Regression_energy: 0.2091 | Regression_eta: 0.0169 | Regression_pt: 0.2150 | Regression_sin_phi: 0.0215 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.5385 | Classification_binary: 4.2137 | Regression_cos_phi: 0.1655 | Regression_energy: 2.2670 | Regression_eta: 0.1344 | Regression_pt: 2.9717 | Regression_sin_phi: 0.2306 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5222s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0025 | max_backward: 0.0413 | max_optimizer_step: 0.0626 | max_tensorboard_logging: 0.1563 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2579.5 | uss_MB: 1413.6 | child_rss_MB: 22606.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 42.3 +rank0 [2026-11-14 05:37:36,094] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:38:03,688] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 05:38:03,688] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:06,487] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 05:39:06,487] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:28,333] INFO : Step 3000/500000 rank0 | Train Loss: 10.6797 | Train Loss Unweighted: 4.5679 | Train Loss Components: Classification: 0.0419 | Classification_binary: 4.0139 | Regression_cos_phi: 0.0129 | Regression_energy: 0.2329 | Regression_eta: 0.0176 | Regression_pt: 0.2338 | Regression_sin_phi: 0.0149 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4663 | Classification_binary: 4.0139 | Regression_cos_phi: 0.1419 | Regression_energy: 2.5249 | Regression_eta: 0.1400 | Regression_pt: 3.2325 | Regression_sin_phi: 0.1603 | LR: 2.05e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.6454s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.2768 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2582.4 | uss_MB: 1416.8 | child_rss_MB: 22610.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 16.3 +rank0 [2026-11-14 05:40:10,220] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 05:40:10,220] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:13,215] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 05:41:13,215] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:21,015] INFO : Step 4000/500000 rank0 | Train Loss: 9.3290 | Train Loss Unweighted: 4.3481 | Train Loss Components: Classification: 0.0463 | Classification_binary: 3.8951 | Regression_cos_phi: 0.0167 | Regression_energy: 0.1852 | Regression_eta: 0.0122 | Regression_pt: 0.1819 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.5156 | Classification_binary: 3.8951 | Regression_cos_phi: 0.1835 | Regression_energy: 2.0074 | Regression_eta: 0.0974 | Regression_pt: 2.5144 | Regression_sin_phi: 0.1154 | LR: 2.08e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5099s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0027 | max_backward: 0.0400 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1275 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2582.6 | uss_MB: 1417.0 | child_rss_MB: 22610.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:41:21,017] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:42:16,228] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 05:42:16,228] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:43:12,674] INFO : Step 5000/500000 rank0 | Train Loss: 11.4198 | Train Loss Unweighted: 4.7503 | Train Loss Components: Classification: 0.0443 | Classification_binary: 4.1483 | Regression_cos_phi: 0.0163 | Regression_energy: 0.2563 | Regression_eta: 0.0146 | Regression_pt: 0.2591 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4936 | Classification_binary: 4.1483 | Regression_cos_phi: 0.1795 | Regression_energy: 2.7779 | Regression_eta: 0.1159 | Regression_pt: 3.5816 | Regression_sin_phi: 0.1230 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5594s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0350 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1840 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2581.7 | uss_MB: 1416.2 | child_rss_MB: 22611.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:43:19,074] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 05:43:19,075] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:21,206] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 05:44:21,206] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:45:03,640] INFO : Step 6000/500000 rank0 | Train Loss: 8.9600 | Train Loss Unweighted: 4.3570 | Train Loss Components: Classification: 0.0434 | Classification_binary: 3.9415 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1661 | Regression_eta: 0.0096 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4832 | Classification_binary: 3.9415 | Regression_cos_phi: 0.0875 | Regression_energy: 1.8010 | Regression_eta: 0.0762 | Regression_pt: 2.4501 | Regression_sin_phi: 0.1206 | LR: 2.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4868s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0746 | max_tensorboard_logging: 0.1284 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2581.8 | uss_MB: 1416.2 | child_rss_MB: 22608.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:45:03,642] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:45:23,811] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 05:45:23,811] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:25,842] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 05:46:25,842] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:54,719] INFO : Step 7000/500000 rank0 | Train Loss: 9.7732 | Train Loss Unweighted: 4.2527 | Train Loss Components: Classification: 0.0470 | Classification_binary: 3.7504 | Regression_cos_phi: 0.0165 | Regression_energy: 0.2044 | Regression_eta: 0.0176 | Regression_pt: 0.2054 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.5230 | Classification_binary: 3.7504 | Regression_cos_phi: 0.1820 | Regression_energy: 2.2159 | Regression_eta: 0.1400 | Regression_pt: 2.8393 | Regression_sin_phi: 0.1227 | LR: 2.26e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5028s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0377 | max_loss: 0.0029 | max_backward: 0.0416 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2583.6 | uss_MB: 1418.1 | child_rss_MB: 22613.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:47:29,122] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 05:47:29,122] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:31,994] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 05:48:31,994] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:47,085] INFO : Step 8000/500000 rank0 | Train Loss: 9.3288 | Train Loss Unweighted: 4.3855 | Train Loss Components: Classification: 0.0356 | Classification_binary: 3.9366 | Regression_cos_phi: 0.0151 | Regression_energy: 0.1869 | Regression_eta: 0.0140 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3963 | Classification_binary: 3.9366 | Regression_cos_phi: 0.1659 | Regression_energy: 2.0256 | Regression_eta: 0.1115 | Regression_pt: 2.5685 | Regression_sin_phi: 0.1245 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5648s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0386 | max_loss: 0.0028 | max_backward: 0.0414 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1818 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2583.8 | uss_MB: 1418.2 | child_rss_MB: 22611.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:48:47,087] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:48:47,156] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:49:34,692] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 05:49:34,692] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:37,071] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 05:50:37,071] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:38,370] INFO : Step 9000/500000 rank0 | Train Loss: 10.4118 | Train Loss Unweighted: 4.4534 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.9188 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2330 | Regression_eta: 0.0090 | Regression_pt: 0.2393 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3348 | Classification_binary: 3.9188 | Regression_cos_phi: 0.1257 | Regression_energy: 2.5254 | Regression_eta: 0.0714 | Regression_pt: 3.3080 | Regression_sin_phi: 0.1277 | LR: 2.43e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4931s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0384 | max_loss: 0.0031 | max_backward: 0.0419 | max_optimizer_step: 0.0620 | max_tensorboard_logging: 0.1228 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2581.4 | uss_MB: 1415.8 | child_rss_MB: 22611.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:51:40,068] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 05:51:40,068] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:30,410] INFO : Step 10000/500000 rank0 | Train Loss: 9.6583 | Train Loss Unweighted: 4.5131 | Train Loss Components: Classification: 0.0349 | Classification_binary: 4.0479 | Regression_cos_phi: 0.0140 | Regression_energy: 0.2010 | Regression_eta: 0.0109 | Regression_pt: 0.1968 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3889 | Classification_binary: 4.0479 | Regression_cos_phi: 0.1543 | Regression_energy: 2.1783 | Regression_eta: 0.0865 | Regression_pt: 2.7205 | Regression_sin_phi: 0.0820 | LR: 2.52e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5027s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0034 | max_backward: 0.0406 | max_optimizer_step: 0.0637 | max_tensorboard_logging: 0.1245 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2581.8 | uss_MB: 1416.2 | child_rss_MB: 22610.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:52:30,411] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:52:30,478] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:52:30,479] INFO : Running validation on rank0 +rank0 [2026-11-14 05:52:30,479] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:52:30,479] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:52:30,684] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 05:52:30,882] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:52:30,965] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:52:31,021] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:52:31,079] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:52:31,129] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:52:31,186] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:52:31,242] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:52:31,292] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:52:31,373] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:52:31,453] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:52:31,507] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:52:31,562] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:52:31,612] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:52:31,666] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:52:31,724] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:52:31,774] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:52:31,778] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:52:31,780] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:52:31,783] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:52:31,785] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:52:31,787] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:52:31,789] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:52:31,791] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:52:31,793] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:52:31,795] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:52:31,797] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:52:31,799] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:52:31,801] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:52:31,803] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:52:31,805] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:52:31,807] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:52:33,386] INFO : VALIDATION | Step=10000/500000 | Train Loss=9.6583 | Valid Loss=9.9378 | Train Loss Unweighted=4.5131 | Valid Loss Unweighted=4.4089 | Stale=0 | ETA=919.6m +rank0 [2026-11-14 05:52:33,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:52:33,836] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:52:33,844] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:33,898] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:52:34,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:52:34,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:52:35,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:52:35,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:52:35,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:52:35,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:52:36,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:52:36,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:52:36,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:52:36,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:52:37,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:52:37,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:52:37,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:52:37,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:52:38,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:52:38,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:52:38,497] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:52:38,505] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:39,095] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:39,858] INFO : Plotted number of elements +rank0 [2026-11-14 05:52:41,251] INFO : Plotted jets +rank0 [2026-11-14 05:52:42,238] INFO : Plotted jet ratio +rank0 [2026-11-14 05:52:55,581] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 05:52:55,582] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:53:57,279] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 05:53:57,279] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:34,425] INFO : Step 11000/500000 rank0 | Train Loss: 9.8952 | Train Loss Unweighted: 4.4012 | Train Loss Components: Classification: 0.0358 | Classification_binary: 3.9067 | Regression_cos_phi: 0.0073 | Regression_energy: 0.2072 | Regression_eta: 0.0117 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3990 | Classification_binary: 3.9067 | Regression_cos_phi: 0.0800 | Regression_energy: 2.2463 | Regression_eta: 0.0932 | Regression_pt: 3.0209 | Regression_sin_phi: 0.1491 | LR: 2.63e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5468s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0386 | max_loss: 0.0032 | max_backward: 0.0411 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1333 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2764.3 | uss_MB: 1613.5 | child_rss_MB: 22625.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:55:00,349] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 05:55:00,349] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:01,065] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 05:56:01,066] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:23,723] INFO : Step 12000/500000 rank0 | Train Loss: 8.4180 | Train Loss Unweighted: 3.9754 | Train Loss Components: Classification: 0.0341 | Classification_binary: 3.5708 | Regression_cos_phi: 0.0179 | Regression_energy: 0.1598 | Regression_eta: 0.0166 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3798 | Classification_binary: 3.5708 | Regression_cos_phi: 0.1974 | Regression_energy: 1.7322 | Regression_eta: 0.1321 | Regression_pt: 2.3009 | Regression_sin_phi: 0.1048 | LR: 2.75e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5349s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1524 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2764.4 | uss_MB: 1613.5 | child_rss_MB: 22622.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:56:23,725] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:56:23,796] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:57:04,065] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 05:57:04,065] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:06,559] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 05:58:06,559] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:15,174] INFO : Step 13000/500000 rank0 | Train Loss: 8.3447 | Train Loss Unweighted: 3.8651 | Train Loss Components: Classification: 0.0274 | Classification_binary: 3.4625 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1676 | Regression_eta: 0.0090 | Regression_pt: 0.1796 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3058 | Classification_binary: 3.4625 | Regression_cos_phi: 0.0890 | Regression_energy: 1.8172 | Regression_eta: 0.0717 | Regression_pt: 2.4828 | Regression_sin_phi: 0.1157 | LR: 2.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5045s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0401 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1290 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2693.3 | uss_MB: 1542.5 | child_rss_MB: 22626.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:59:09,470] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:59:09,470] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:06,846] INFO : Step 14000/500000 rank0 | Train Loss: 7.7032 | Train Loss Unweighted: 3.8688 | Train Loss Components: Classification: 0.0383 | Classification_binary: 3.5226 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1417 | Regression_eta: 0.0055 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4267 | Classification_binary: 3.5226 | Regression_cos_phi: 0.1060 | Regression_energy: 1.5355 | Regression_eta: 0.0439 | Regression_pt: 1.9978 | Regression_sin_phi: 0.0707 | LR: 3.02e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5553s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0029 | max_backward: 0.0399 | max_optimizer_step: 0.0768 | max_tensorboard_logging: 0.1569 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2693.6 | uss_MB: 1542.8 | child_rss_MB: 22624.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:00:06,848] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:00:06,920] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 06:00:12,479] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 06:00:12,479] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:15,424] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 06:01:15,424] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:59,742] INFO : Step 15000/500000 rank0 | Train Loss: 8.3672 | Train Loss Unweighted: 4.1495 | Train Loss Components: Classification: 0.0321 | Classification_binary: 3.7667 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1615 | Regression_eta: 0.0120 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3578 | Classification_binary: 3.7667 | Regression_cos_phi: 0.1269 | Regression_energy: 1.7504 | Regression_eta: 0.0954 | Regression_pt: 2.1981 | Regression_sin_phi: 0.0720 | LR: 3.17e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5511s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0389 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1602 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2696.3 | uss_MB: 1545.5 | child_rss_MB: 22625.4 | allocated_MB: 126.1 | reserved_MB: 650.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:02:19,183] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 06:02:19,183] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:21,441] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 06:03:21,441] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:51,077] INFO : Step 16000/500000 rank0 | Train Loss: 8.2910 | Train Loss Unweighted: 4.0729 | Train Loss Components: Classification: 0.0317 | Classification_binary: 3.6916 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1634 | Regression_eta: 0.0096 | Regression_pt: 0.1619 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3533 | Classification_binary: 3.6916 | Regression_cos_phi: 0.0912 | Regression_energy: 1.7716 | Regression_eta: 0.0760 | Regression_pt: 2.2384 | Regression_sin_phi: 0.0688 | LR: 3.33e-05 | DataLoad Time: 0.0051s | Model Forward Time: 0.5303s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0382 | max_loss: 0.0029 | max_backward: 0.0431 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1532 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2696.3 | uss_MB: 1545.4 | child_rss_MB: 22628.0 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:03:51,078] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:03:51,151] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 06:04:24,287] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 06:04:24,287] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:26,712] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 06:05:26,712] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:42,723] INFO : Step 17000/500000 rank0 | Train Loss: 8.8821 | Train Loss Unweighted: 4.0547 | Train Loss Components: Classification: 0.0384 | Classification_binary: 3.6183 | Regression_cos_phi: 0.0203 | Regression_energy: 0.1804 | Regression_eta: 0.0068 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4280 | Classification_binary: 3.6183 | Regression_cos_phi: 0.2234 | Regression_energy: 1.9550 | Regression_eta: 0.0539 | Regression_pt: 2.4962 | Regression_sin_phi: 0.1074 | LR: 3.51e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5111s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0691 | max_tensorboard_logging: 0.1235 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2692.9 | uss_MB: 1542.0 | child_rss_MB: 22628.1 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:30,184] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 06:06:30,184] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:32,227] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 06:07:32,227] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:34,426] INFO : Step 18000/500000 rank0 | Train Loss: 9.1418 | Train Loss Unweighted: 4.2839 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.8441 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1889 | Regression_eta: 0.0119 | Regression_pt: 0.1855 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3774 | Classification_binary: 3.8441 | Regression_cos_phi: 0.1143 | Regression_energy: 2.0474 | Regression_eta: 0.0949 | Regression_pt: 2.5645 | Regression_sin_phi: 0.0990 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.5296s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0026 | max_backward: 0.0407 | max_optimizer_step: 0.0650 | max_tensorboard_logging: 0.1525 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2693.7 | uss_MB: 1542.9 | child_rss_MB: 22635.6 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 06:07:34,428] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:07:34,498] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 06:08:35,008] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 06:08:35,008] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:26,167] INFO : Step 19000/500000 rank0 | Train Loss: 10.5853 | Train Loss Unweighted: 4.2671 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.6971 | Regression_cos_phi: 0.0100 | Regression_energy: 0.2471 | Regression_eta: 0.0148 | Regression_pt: 0.2486 | Regression_sin_phi: 0.0156 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3774 | Classification_binary: 3.6971 | Regression_cos_phi: 0.1099 | Regression_energy: 2.6790 | Regression_eta: 0.1182 | Regression_pt: 3.4369 | Regression_sin_phi: 0.1668 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5558s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0028 | max_backward: 0.0392 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1737 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2696.2 | uss_MB: 1545.4 | child_rss_MB: 22633.8 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 06:09:38,103] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 06:09:38,103] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:10:40,925] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 06:10:40,925] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:18,831] INFO : Step 20000/500000 rank0 | Train Loss: 9.0492 | Train Loss Unweighted: 4.1925 | Train Loss Components: Classification: 0.0308 | Classification_binary: 3.7550 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1878 | Regression_eta: 0.0083 | Regression_pt: 0.1897 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3427 | Classification_binary: 3.7550 | Regression_cos_phi: 0.1187 | Regression_energy: 2.0355 | Regression_eta: 0.0660 | Regression_pt: 2.6227 | Regression_sin_phi: 0.1085 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5041s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0028 | max_backward: 0.0418 | max_optimizer_step: 0.0638 | max_tensorboard_logging: 0.1216 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2696.3 | uss_MB: 1545.5 | child_rss_MB: 22630.0 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:11:18,833] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:11:18,904] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:11:18,905] INFO : Running validation on rank0 +rank0 [2026-11-14 06:11:18,905] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:11:18,905] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:11:19,055] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 06:11:19,180] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:11:19,301] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:11:19,372] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:11:19,441] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:11:19,509] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:11:19,578] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:11:19,648] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:11:19,749] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:11:19,869] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:11:19,989] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:11:20,059] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:11:20,129] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:11:20,198] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:11:20,268] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:11:20,337] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:11:20,449] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:11:20,456] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:11:20,458] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:11:20,460] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:11:20,462] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:11:20,464] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:11:20,467] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:11:20,469] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:11:20,471] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:11:20,473] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:11:20,475] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:11:20,477] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:11:20,479] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:11:20,481] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:11:20,483] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:11:20,485] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:11:22,117] INFO : VALIDATION | Step=20000/500000 | Train Loss=9.0492 | Valid Loss=9.5361 | Train Loss Unweighted=4.1925 | Valid Loss Unweighted=4.2354 | Stale=0 | ETA=901.9m +rank0 [2026-11-14 06:11:22,131] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:11:22,501] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:11:22,536] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:22,587] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:11:23,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:11:23,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:11:23,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:11:23,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:11:24,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:11:24,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:11:24,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:11:24,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:11:25,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:11:25,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:11:25,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:11:25,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:11:27,482] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:11:27,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:11:27,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:11:28,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:11:28,282] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-14 06:11:28,290] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:28,888] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:29,293] INFO : Plotted number of elements +rank0 [2026-11-14 06:11:30,645] INFO : Plotted jets +rank0 [2026-11-14 06:11:31,649] INFO : Plotted jet ratio +rank0 [2026-11-14 06:11:58,086] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 06:11:58,086] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:01,031] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 06:13:01,031] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:24,597] INFO : Step 21000/500000 rank0 | Train Loss: 9.7959 | Train Loss Unweighted: 4.4097 | Train Loss Components: Classification: 0.0235 | Classification_binary: 3.9260 | Regression_cos_phi: 0.0079 | Regression_energy: 0.2085 | Regression_eta: 0.0134 | Regression_pt: 0.2204 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2622 | Classification_binary: 3.9260 | Regression_cos_phi: 0.0871 | Regression_energy: 2.2597 | Regression_eta: 0.1066 | Regression_pt: 3.0463 | Regression_sin_phi: 0.1081 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5436s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0031 | max_backward: 0.0414 | max_optimizer_step: 0.0724 | max_tensorboard_logging: 0.1440 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2726.7 | uss_MB: 1576.3 | child_rss_MB: 22630.2 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:14:04,381] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 06:14:04,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:07,378] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 06:15:07,378] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:16,985] INFO : Step 22000/500000 rank0 | Train Loss: 8.6585 | Train Loss Unweighted: 4.1771 | Train Loss Components: Classification: 0.0312 | Classification_binary: 3.7726 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1741 | Regression_eta: 0.0072 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3471 | Classification_binary: 3.7726 | Regression_cos_phi: 0.0777 | Regression_energy: 1.8870 | Regression_eta: 0.0575 | Regression_pt: 2.3778 | Regression_sin_phi: 0.1388 | LR: 4.50e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5249s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0382 | max_loss: 0.0029 | max_backward: 0.0431 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1318 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2727.6 | uss_MB: 1577.3 | child_rss_MB: 22630.6 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:15:16,987] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:15:17,061] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:16:10,833] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 06:16:10,833] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:09,549] INFO : Step 23000/500000 rank0 | Train Loss: 7.5217 | Train Loss Unweighted: 3.7963 | Train Loss Components: Classification: 0.0289 | Classification_binary: 3.4619 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1426 | Regression_eta: 0.0046 | Regression_pt: 0.1475 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3214 | Classification_binary: 3.4619 | Regression_cos_phi: 0.0545 | Regression_energy: 1.5459 | Regression_eta: 0.0367 | Regression_pt: 2.0388 | Regression_sin_phi: 0.0625 | LR: 4.73e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5229s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0031 | max_backward: 0.0418 | max_optimizer_step: 0.0705 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.6 | uss_MB: 1580.3 | child_rss_MB: 22643.6 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:17:14,245] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 06:17:14,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:16,495] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 06:18:16,495] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:19:02,043] INFO : Step 24000/500000 rank0 | Train Loss: 7.5216 | Train Loss Unweighted: 3.9589 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.6326 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1312 | Regression_eta: 0.0130 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0142 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3305 | Classification_binary: 3.6326 | Regression_cos_phi: 0.1229 | Regression_energy: 1.4221 | Regression_eta: 0.1036 | Regression_pt: 1.7575 | Regression_sin_phi: 0.1524 | LR: 4.97e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4692s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0026 | max_backward: 0.0395 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1278 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.7 | uss_MB: 1580.3 | child_rss_MB: 22641.2 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:19:02,045] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:19:02,114] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:19:20,143] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 06:19:20,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:21,608] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 06:20:21,608] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:53,021] INFO : Step 25000/500000 rank0 | Train Loss: 8.1193 | Train Loss Unweighted: 3.8959 | Train Loss Components: Classification: 0.0344 | Classification_binary: 3.5117 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1620 | Regression_eta: 0.0132 | Regression_pt: 0.1573 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3835 | Classification_binary: 3.5117 | Regression_cos_phi: 0.1156 | Regression_energy: 1.7564 | Regression_eta: 0.1050 | Regression_pt: 2.1750 | Regression_sin_phi: 0.0721 | LR: 5.22e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5495s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0704 | max_tensorboard_logging: 0.1606 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2726.7 | uss_MB: 1576.4 | child_rss_MB: 22638.9 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:21:25,409] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 06:21:25,409] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:28,240] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 06:22:28,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:45,286] INFO : Step 26000/500000 rank0 | Train Loss: 8.4002 | Train Loss Unweighted: 3.9523 | Train Loss Components: Classification: 0.0356 | Classification_binary: 3.5510 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1681 | Regression_eta: 0.0058 | Regression_pt: 0.1690 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3966 | Classification_binary: 3.5510 | Regression_cos_phi: 0.1176 | Regression_energy: 1.8220 | Regression_eta: 0.0462 | Regression_pt: 2.3365 | Regression_sin_phi: 0.1304 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5497s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0030 | max_backward: 0.0427 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1547 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2727.3 | uss_MB: 1577.0 | child_rss_MB: 22637.3 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:22:45,288] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:22:45,362] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:23:31,775] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 06:23:31,775] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:34,683] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 06:24:34,683] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:37,795] INFO : Step 27000/500000 rank0 | Train Loss: 8.0476 | Train Loss Unweighted: 3.7336 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.3428 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1654 | Regression_eta: 0.0118 | Regression_pt: 0.1655 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3254 | Classification_binary: 3.3428 | Regression_cos_phi: 0.1034 | Regression_energy: 1.7933 | Regression_eta: 0.0942 | Regression_pt: 2.2884 | Regression_sin_phi: 0.1001 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5162s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0366 | max_loss: 0.0027 | max_backward: 0.0400 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1343 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.6 | uss_MB: 1580.3 | child_rss_MB: 22637.9 | allocated_MB: 126.1 | reserved_MB: 632.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:25:38,079] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 06:25:38,079] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:29,989] INFO : Step 28000/500000 rank0 | Train Loss: 8.7899 | Train Loss Unweighted: 3.9411 | Train Loss Components: Classification: 0.0345 | Classification_binary: 3.5046 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1879 | Regression_eta: 0.0060 | Regression_pt: 0.1876 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3843 | Classification_binary: 3.5046 | Regression_cos_phi: 0.1118 | Regression_energy: 2.0371 | Regression_eta: 0.0476 | Regression_pt: 2.5938 | Regression_sin_phi: 0.1108 | LR: 6.01e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5673s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0378 | max_loss: 0.0032 | max_backward: 0.0423 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.1767 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.8 | uss_MB: 1580.5 | child_rss_MB: 22643.4 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:26:29,991] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:26:30,061] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:26:41,213] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 06:26:41,213] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:27:44,126] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 06:27:44,126] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:22,153] INFO : Step 29000/500000 rank0 | Train Loss: 8.6877 | Train Loss Unweighted: 4.1204 | Train Loss Components: Classification: 0.0293 | Classification_binary: 3.7091 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1763 | Regression_eta: 0.0098 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3267 | Classification_binary: 3.7091 | Regression_cos_phi: 0.0883 | Regression_energy: 1.9106 | Regression_eta: 0.0784 | Regression_pt: 2.4977 | Regression_sin_phi: 0.0769 | LR: 6.29e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5206s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0383 | max_loss: 0.0031 | max_backward: 0.0424 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1355 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2731.3 | uss_MB: 1581.0 | child_rss_MB: 22643.6 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:28:47,068] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 06:28:47,068] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:29:49,850] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 06:29:49,850] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:14,495] INFO : Step 30000/500000 rank0 | Train Loss: 8.1003 | Train Loss Unweighted: 3.7425 | Train Loss Components: Classification: 0.0373 | Classification_binary: 3.3508 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1683 | Regression_eta: 0.0048 | Regression_pt: 0.1696 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4156 | Classification_binary: 3.3508 | Regression_cos_phi: 0.0709 | Regression_energy: 1.8241 | Regression_eta: 0.0381 | Regression_pt: 2.3442 | Regression_sin_phi: 0.0565 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5199s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0420 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2727.6 | uss_MB: 1577.3 | child_rss_MB: 22648.0 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:30:14,496] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:30:14,567] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:30:14,567] INFO : Running validation on rank0 +rank0 [2026-11-14 06:30:14,568] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:30:14,568] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:30:14,731] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 06:30:14,853] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:30:14,968] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:30:15,036] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:30:15,102] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:30:15,169] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:30:15,237] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:30:15,305] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:30:15,406] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:30:15,523] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:30:15,635] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:30:15,700] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:30:15,765] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:30:15,830] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:30:15,898] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:30:15,966] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:30:16,081] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:30:16,088] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:30:16,090] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:30:16,093] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:30:16,095] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:30:16,097] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:30:16,099] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:30:16,101] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:30:16,103] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:30:16,105] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:30:16,107] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:30:16,109] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:30:16,111] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:30:16,113] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:30:16,114] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:30:16,117] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:30:17,690] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.1003 | Valid Loss=9.2597 | Train Loss Unweighted=3.7425 | Valid Loss Unweighted=4.1063 | Stale=0 | ETA=885.2m +rank0 [2026-11-14 06:30:17,691] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:30:17,768] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:30:17,776] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:17,828] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:30:18,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:30:18,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:30:18,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:30:19,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:30:19,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:30:19,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:30:19,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:30:20,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:30:20,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:30:20,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:30:20,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:30:21,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:30:21,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:30:21,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:30:21,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:30:22,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:30:22,266] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:30:22,273] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:22,846] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:23,258] INFO : Plotted number of elements +rank0 [2026-11-14 06:30:24,616] INFO : Plotted jets +rank0 [2026-11-14 06:30:25,586] INFO : Plotted jet ratio +rank0 [2026-11-14 06:31:04,970] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 06:31:04,970] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:08,069] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 06:32:08,070] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:18,609] INFO : Step 31000/500000 rank0 | Train Loss: 8.9334 | Train Loss Unweighted: 4.3159 | Train Loss Components: Classification: 0.0311 | Classification_binary: 3.9006 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1793 | Regression_eta: 0.0067 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3469 | Classification_binary: 3.9006 | Regression_cos_phi: 0.0915 | Regression_energy: 1.9433 | Regression_eta: 0.0530 | Regression_pt: 2.5057 | Regression_sin_phi: 0.0924 | LR: 6.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5148s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0028 | max_backward: 0.0409 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1210 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2860.5 | uss_MB: 1710.5 | child_rss_MB: 22632.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:33:11,333] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 06:33:11,333] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:10,421] INFO : Step 32000/500000 rank0 | Train Loss: 8.0630 | Train Loss Unweighted: 3.7867 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.4000 | Regression_cos_phi: 0.0170 | Regression_energy: 0.1606 | Regression_eta: 0.0074 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3314 | Classification_binary: 3.4000 | Regression_cos_phi: 0.1875 | Regression_energy: 1.7405 | Regression_eta: 0.0592 | Regression_pt: 2.2311 | Regression_sin_phi: 0.1134 | LR: 7.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5436s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0027 | max_backward: 0.0444 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1529 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2860.6 | uss_MB: 1710.5 | child_rss_MB: 22625.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:34:10,423] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:34:10,491] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:34:14,258] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 06:34:14,258] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:17,412] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 06:35:17,412] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:02,839] INFO : Step 33000/500000 rank0 | Train Loss: 7.7284 | Train Loss Unweighted: 3.7381 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.3782 | Regression_cos_phi: 0.0046 | Regression_energy: 0.1524 | Regression_eta: 0.0101 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2922 | Classification_binary: 3.3782 | Regression_cos_phi: 0.0512 | Regression_energy: 1.6524 | Regression_eta: 0.0806 | Regression_pt: 2.1812 | Regression_sin_phi: 0.0926 | LR: 7.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5249s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0382 | max_loss: 0.0031 | max_backward: 0.0409 | max_optimizer_step: 0.0697 | max_tensorboard_logging: 0.1323 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.2 | uss_MB: 1604.2 | child_rss_MB: 22629.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:36:20,409] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 06:36:20,410] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:22,896] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 06:37:22,896] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:54,435] INFO : Step 34000/500000 rank0 | Train Loss: 9.4831 | Train Loss Unweighted: 4.2166 | Train Loss Components: Classification: 0.0320 | Classification_binary: 3.7427 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2062 | Regression_eta: 0.0112 | Regression_pt: 0.2094 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3567 | Classification_binary: 3.7427 | Regression_cos_phi: 0.0893 | Regression_energy: 2.2347 | Regression_eta: 0.0894 | Regression_pt: 2.8948 | Regression_sin_phi: 0.0754 | LR: 7.83e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5203s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0387 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1294 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.9 | uss_MB: 1604.8 | child_rss_MB: 22628.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:37:54,437] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:37:54,503] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:38:25,716] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 06:38:25,716] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:39:27,887] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 06:39:27,887] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:39:45,937] INFO : Step 35000/500000 rank0 | Train Loss: 8.7704 | Train Loss Unweighted: 3.9651 | Train Loss Components: Classification: 0.0465 | Classification_binary: 3.5310 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1791 | Regression_eta: 0.0094 | Regression_pt: 0.1831 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.5175 | Classification_binary: 3.5310 | Regression_cos_phi: 0.0871 | Regression_energy: 1.9415 | Regression_eta: 0.0747 | Regression_pt: 2.5315 | Regression_sin_phi: 0.0871 | LR: 8.16e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5338s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0396 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1384 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2757.9 | uss_MB: 1607.9 | child_rss_MB: 22631.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 20.3 +rank0 [2026-11-14 06:40:30,968] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 06:40:30,968] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:41:33,394] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 06:41:33,394] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:41:37,205] INFO : Step 36000/500000 rank0 | Train Loss: 7.4502 | Train Loss Unweighted: 3.5235 | Train Loss Components: Classification: 0.0345 | Classification_binary: 3.1665 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1437 | Regression_eta: 0.0135 | Regression_pt: 0.1484 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3844 | Classification_binary: 3.1665 | Regression_cos_phi: 0.0930 | Regression_energy: 1.5577 | Regression_eta: 0.1076 | Regression_pt: 2.0509 | Regression_sin_phi: 0.0900 | LR: 8.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4839s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0028 | max_backward: 0.0400 | max_optimizer_step: 0.0615 | max_tensorboard_logging: 0.1415 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2758.0 | uss_MB: 1608.0 | child_rss_MB: 22633.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:41:37,207] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:41:37,273] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:42:35,961] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 06:42:35,961] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:43:29,432] INFO : Step 37000/500000 rank0 | Train Loss: 8.5854 | Train Loss Unweighted: 3.8032 | Train Loss Components: Classification: 0.0361 | Classification_binary: 3.3700 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1897 | Regression_eta: 0.0120 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4020 | Classification_binary: 3.3700 | Regression_cos_phi: 0.0633 | Regression_energy: 2.0561 | Regression_eta: 0.0958 | Regression_pt: 2.5174 | Regression_sin_phi: 0.0808 | LR: 8.85e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5171s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0401 | max_optimizer_step: 0.0616 | max_tensorboard_logging: 0.1290 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2758.1 | uss_MB: 1608.1 | child_rss_MB: 22636.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 30.3 +rank0 [2026-11-14 06:43:39,268] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 06:43:39,268] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:44:41,236] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 06:44:41,236] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:45:20,268] INFO : Step 38000/500000 rank0 | Train Loss: 7.6476 | Train Loss Unweighted: 3.7467 | Train Loss Components: Classification: 0.0385 | Classification_binary: 3.3931 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1462 | Regression_eta: 0.0081 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4283 | Classification_binary: 3.3931 | Regression_cos_phi: 0.0916 | Regression_energy: 1.5848 | Regression_eta: 0.0646 | Regression_pt: 2.0025 | Regression_sin_phi: 0.0828 | LR: 9.21e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5593s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0030 | max_backward: 0.0401 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1689 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.9 | uss_MB: 1604.9 | child_rss_MB: 22640.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:45:20,270] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:45:20,338] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:45:44,430] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 06:45:44,430] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:46:46,963] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 06:46:46,963] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:12,061] INFO : Step 39000/500000 rank0 | Train Loss: 7.3042 | Train Loss Unweighted: 3.6422 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.3119 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1365 | Regression_eta: 0.0071 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3138 | Classification_binary: 3.3119 | Regression_cos_phi: 0.1022 | Regression_energy: 1.4797 | Regression_eta: 0.0564 | Regression_pt: 1.9571 | Regression_sin_phi: 0.0831 | LR: 9.57e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5734s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0029 | max_backward: 0.0408 | max_optimizer_step: 0.0683 | max_tensorboard_logging: 0.1884 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2758.1 | uss_MB: 1608.1 | child_rss_MB: 22642.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:47:49,745] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 06:47:49,746] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:48:52,364] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 06:48:52,364] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:03,780] INFO : Step 40000/500000 rank0 | Train Loss: 8.4305 | Train Loss Unweighted: 3.9832 | Train Loss Components: Classification: 0.0350 | Classification_binary: 3.5788 | Regression_cos_phi: 0.0142 | Regression_energy: 0.1621 | Regression_eta: 0.0129 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3895 | Classification_binary: 3.5788 | Regression_cos_phi: 0.1560 | Regression_energy: 1.7571 | Regression_eta: 0.1026 | Regression_pt: 2.2916 | Regression_sin_phi: 0.1548 | LR: 9.94e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5415s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0034 | max_backward: 0.0409 | max_optimizer_step: 0.0798 | max_tensorboard_logging: 0.1353 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2758.4 | uss_MB: 1608.4 | child_rss_MB: 22638.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:49:03,782] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 06:49:03,854] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:49:03,856] INFO : Running validation on rank0 +rank0 [2026-11-14 06:49:03,856] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:49:03,856] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:49:04,000] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 06:49:04,121] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:49:04,237] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:49:04,305] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:49:04,371] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:49:04,436] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:49:04,502] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:49:04,568] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:49:04,671] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:49:04,791] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:49:04,908] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:49:04,976] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:49:05,042] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:49:05,108] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:49:05,177] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:49:05,245] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:49:05,346] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:49:05,353] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:49:05,355] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:49:05,357] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:49:05,359] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:49:05,361] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:49:05,363] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:49:05,365] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:49:05,367] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:49:05,370] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:49:05,371] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:49:05,373] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:49:05,376] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:49:05,378] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:49:05,380] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:49:05,382] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:49:06,939] INFO : VALIDATION | Step=40000/500000 | Train Loss=8.4305 | Valid Loss=8.8503 | Train Loss Unweighted=3.9832 | Valid Loss Unweighted=3.9182 | Stale=0 | ETA=866.2m +rank0 [2026-11-14 06:49:06,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:49:07,019] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:49:07,038] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:07,087] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:49:07,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:49:07,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:49:08,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:49:08,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:49:08,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:49:08,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:49:09,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:49:09,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:49:09,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:49:09,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:49:10,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:49:10,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:49:10,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:49:10,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:49:11,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:49:11,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:49:11,464] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:49:11,473] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:12,053] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:12,447] INFO : Plotted number of elements +rank0 [2026-11-14 06:49:13,792] INFO : Plotted jets +rank0 [2026-11-14 06:49:14,779] INFO : Plotted jet ratio +rank0 [2026-11-14 06:50:06,907] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 06:50:06,907] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:51:06,959] INFO : Step 41000/500000 rank0 | Train Loss: 7.6717 | Train Loss Unweighted: 3.6997 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.3427 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1551 | Regression_eta: 0.0041 | Regression_pt: 0.1553 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2851 | Classification_binary: 3.3427 | Regression_cos_phi: 0.0632 | Regression_energy: 1.6817 | Regression_eta: 0.0327 | Regression_pt: 2.1473 | Regression_sin_phi: 0.1190 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5687s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1849 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2778.5 | uss_MB: 1628.5 | child_rss_MB: 22644.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:51:09,835] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 06:51:09,835] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:52:12,397] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 06:52:12,397] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:52:58,548] INFO : Step 42000/500000 rank0 | Train Loss: 7.3540 | Train Loss Unweighted: 3.6255 | Train Loss Components: Classification: 0.0309 | Classification_binary: 3.2874 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1436 | Regression_eta: 0.0077 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3439 | Classification_binary: 3.2874 | Regression_cos_phi: 0.1046 | Regression_energy: 1.5570 | Regression_eta: 0.0617 | Regression_pt: 1.9168 | Regression_sin_phi: 0.0826 | LR: 1.07e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5639s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0390 | max_loss: 0.0027 | max_backward: 0.0401 | max_optimizer_step: 0.0627 | max_tensorboard_logging: 0.1836 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2779.1 | uss_MB: 1629.1 | child_rss_MB: 22643.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:52:58,550] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:52:58,621] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:53:15,336] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 06:53:15,336] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:54:18,129] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 06:54:18,130] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:54:50,715] INFO : Step 43000/500000 rank0 | Train Loss: 7.9896 | Train Loss Unweighted: 3.8364 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.4627 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1570 | Regression_eta: 0.0080 | Regression_pt: 0.1642 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3181 | Classification_binary: 3.4627 | Regression_cos_phi: 0.1010 | Regression_energy: 1.7018 | Regression_eta: 0.0634 | Regression_pt: 2.2702 | Regression_sin_phi: 0.0724 | LR: 1.11e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5135s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1246 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2781.3 | uss_MB: 1631.3 | child_rss_MB: 22646.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:55:21,339] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 06:55:21,340] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:56:24,344] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 06:56:24,344] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:56:43,175] INFO : Step 44000/500000 rank0 | Train Loss: 8.1057 | Train Loss Unweighted: 3.6378 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.2347 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1725 | Regression_eta: 0.0084 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3141 | Classification_binary: 3.2347 | Regression_cos_phi: 0.1314 | Regression_energy: 1.8698 | Regression_eta: 0.0668 | Regression_pt: 2.3937 | Regression_sin_phi: 0.0951 | LR: 1.15e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5865s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0031 | max_backward: 0.0414 | max_optimizer_step: 0.0687 | max_tensorboard_logging: 0.1829 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.7 | uss_MB: 1631.7 | child_rss_MB: 22642.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:56:43,177] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:56:43,249] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:57:27,816] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 06:57:27,816] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:58:31,270] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 06:58:31,271] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:58:36,160] INFO : Step 45000/500000 rank0 | Train Loss: 7.0141 | Train Loss Unweighted: 3.4341 | Train Loss Components: Classification: 0.0258 | Classification_binary: 3.1068 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1350 | Regression_eta: 0.0147 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2879 | Classification_binary: 3.1068 | Regression_cos_phi: 0.1120 | Regression_energy: 1.4632 | Regression_eta: 0.1170 | Regression_pt: 1.8226 | Regression_sin_phi: 0.1045 | LR: 1.19e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5375s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0380 | max_loss: 0.0029 | max_backward: 0.0435 | max_optimizer_step: 0.0687 | max_tensorboard_logging: 0.1436 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.7 | uss_MB: 1631.7 | child_rss_MB: 22646.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:59:34,512] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 06:59:34,512] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:00:28,129] INFO : Step 46000/500000 rank0 | Train Loss: 6.7171 | Train Loss Unweighted: 3.2995 | Train Loss Components: Classification: 0.0307 | Classification_binary: 2.9890 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1240 | Regression_eta: 0.0077 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3417 | Classification_binary: 2.9890 | Regression_cos_phi: 0.1064 | Regression_energy: 1.3443 | Regression_eta: 0.0610 | Regression_pt: 1.7397 | Regression_sin_phi: 0.1349 | LR: 1.23e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5308s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0025 | max_backward: 0.0390 | max_optimizer_step: 0.0628 | max_tensorboard_logging: 0.1557 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2779.3 | uss_MB: 1629.3 | child_rss_MB: 22639.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:00:28,131] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:00:28,201] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 07:00:37,543] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 07:00:37,543] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:01:40,152] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 07:01:40,152] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:02:19,981] INFO : Step 47000/500000 rank0 | Train Loss: 7.8845 | Train Loss Unweighted: 3.8084 | Train Loss Components: Classification: 0.0280 | Classification_binary: 3.4423 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1608 | Regression_eta: 0.0048 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3123 | Classification_binary: 3.4423 | Regression_cos_phi: 0.0562 | Regression_energy: 1.7427 | Regression_eta: 0.0386 | Regression_pt: 2.2193 | Regression_sin_phi: 0.0730 | LR: 1.27e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5422s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0029 | max_backward: 0.0411 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1473 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.4 | uss_MB: 1631.4 | child_rss_MB: 22650.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:02:43,151] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 07:02:43,152] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:03:46,294] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 07:03:46,294] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:12,418] INFO : Step 48000/500000 rank0 | Train Loss: 7.0900 | Train Loss Unweighted: 3.5414 | Train Loss Components: Classification: 0.0273 | Classification_binary: 3.2203 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1316 | Regression_eta: 0.0092 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3039 | Classification_binary: 3.2203 | Regression_cos_phi: 0.0923 | Regression_energy: 1.4270 | Regression_eta: 0.0733 | Regression_pt: 1.8820 | Regression_sin_phi: 0.0913 | LR: 1.31e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.5385s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0374 | max_loss: 0.0030 | max_backward: 0.0410 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1340 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.4 | uss_MB: 1631.4 | child_rss_MB: 22645.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:04:12,420] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:04:12,486] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 07:04:49,751] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 07:04:49,752] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:05:52,443] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 07:05:52,444] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:04,387] INFO : Step 49000/500000 rank0 | Train Loss: 7.3908 | Train Loss Unweighted: 3.7114 | Train Loss Components: Classification: 0.0290 | Classification_binary: 3.3788 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1379 | Regression_eta: 0.0068 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3225 | Classification_binary: 3.3788 | Regression_cos_phi: 0.1090 | Regression_energy: 1.4948 | Regression_eta: 0.0544 | Regression_pt: 1.9334 | Regression_sin_phi: 0.0978 | LR: 1.36e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4789s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0392 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1403 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.5 | uss_MB: 1631.5 | child_rss_MB: 22646.2 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:06:54,667] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 07:06:54,667] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:07:57,320] INFO : Step 50000/500000 rank0 | Train Loss: 7.7648 | Train Loss Unweighted: 3.7156 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.3485 | Regression_cos_phi: 0.0129 | Regression_energy: 0.1503 | Regression_eta: 0.0120 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2771 | Classification_binary: 3.3485 | Regression_cos_phi: 0.1426 | Regression_energy: 1.6292 | Regression_eta: 0.0959 | Regression_pt: 2.1451 | Regression_sin_phi: 0.1263 | LR: 1.40e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5204s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0031 | max_backward: 0.0406 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2778.7 | uss_MB: 1628.7 | child_rss_MB: 22653.9 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:07:57,322] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:07:57,392] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 07:07:57,393] INFO : Running validation on rank0 +rank0 [2026-11-14 07:07:57,393] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:07:57,393] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:07:57,540] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 07:07:57,662] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:07:57,779] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:07:57,848] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:07:57,913] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:07:57,977] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:07:58,042] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:07:58,109] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:07:58,210] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:07:58,326] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:07:58,439] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:07:58,507] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:07:58,572] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:07:58,638] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:07:58,705] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:07:58,773] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:07:58,872] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:07:58,879] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:07:58,882] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:07:58,884] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:07:58,886] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:07:58,888] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:07:58,890] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:07:58,892] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:07:58,894] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:07:58,896] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:07:58,898] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:07:58,900] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:07:58,902] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:07:58,904] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:07:58,906] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:07:58,908] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:08:00,419] INFO : VALIDATION | Step=50000/500000 | Train Loss=7.7648 | Valid Loss=8.8123 | Train Loss Unweighted=3.7156 | Valid Loss Unweighted=3.8453 | Stale=0 | ETA=848.0m +rank0 [2026-11-14 07:08:00,419] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:08:00,772] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:08:00,779] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:00,832] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:08:01,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:08:01,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:08:01,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:08:02,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:08:02,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:08:02,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:08:02,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:08:03,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:08:03,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:08:03,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:08:03,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:08:04,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:08:04,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:08:04,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:08:04,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:08:05,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:08:05,154] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:08:05,161] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:05,736] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:06,116] INFO : Plotted number of elements +rank0 [2026-11-14 07:08:07,416] INFO : Plotted jets +rank0 [2026-11-14 07:08:08,381] INFO : Plotted jet ratio +rank0 [2026-11-14 07:08:10,827] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 07:08:10,827] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:09:12,481] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 07:09:12,481] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:09:59,802] INFO : Step 51000/500000 rank0 | Train Loss: 7.2707 | Train Loss Unweighted: 3.6398 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.3106 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1393 | Regression_eta: 0.0097 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2827 | Classification_binary: 3.3106 | Regression_cos_phi: 0.0746 | Regression_energy: 1.5105 | Regression_eta: 0.0776 | Regression_pt: 1.9106 | Regression_sin_phi: 0.1040 | LR: 1.44e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5191s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0028 | max_backward: 0.0404 | max_optimizer_step: 0.0660 | max_tensorboard_logging: 0.1328 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2904.0 | uss_MB: 1754.1 | child_rss_MB: 22647.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:10:15,750] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 07:10:15,750] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:11:18,819] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 07:11:18,819] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:11:52,381] INFO : Step 52000/500000 rank0 | Train Loss: 7.1306 | Train Loss Unweighted: 3.5259 | Train Loss Components: Classification: 0.0269 | Classification_binary: 3.2004 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1330 | Regression_eta: 0.0089 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2998 | Classification_binary: 3.2004 | Regression_cos_phi: 0.0878 | Regression_energy: 1.4419 | Regression_eta: 0.0707 | Regression_pt: 1.9411 | Regression_sin_phi: 0.0887 | LR: 1.49e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5264s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0029 | max_backward: 0.0412 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.1251 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2904.0 | uss_MB: 1754.1 | child_rss_MB: 22647.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:11:52,383] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:11:52,450] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:12:22,193] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 07:12:22,193] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:13:25,158] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 07:13:25,159] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:13:44,769] INFO : Step 53000/500000 rank0 | Train Loss: 6.1628 | Train Loss Unweighted: 3.0686 | Train Loss Components: Classification: 0.0298 | Classification_binary: 2.7870 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1152 | Regression_eta: 0.0094 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3316 | Classification_binary: 2.7870 | Regression_cos_phi: 0.0701 | Regression_energy: 1.2489 | Regression_eta: 0.0747 | Regression_pt: 1.5811 | Regression_sin_phi: 0.0695 | LR: 1.53e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5572s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0416 | max_loss: 0.0030 | max_backward: 0.0411 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1523 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2904.1 | uss_MB: 1754.1 | child_rss_MB: 22647.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:14:28,468] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 07:14:28,468] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:15:31,423] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 07:15:31,423] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:15:37,249] INFO : Step 54000/500000 rank0 | Train Loss: 6.3708 | Train Loss Unweighted: 3.3509 | Train Loss Components: Classification: 0.0304 | Classification_binary: 3.0768 | Regression_cos_phi: 0.0043 | Regression_energy: 0.1117 | Regression_eta: 0.0081 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3391 | Classification_binary: 3.0768 | Regression_cos_phi: 0.0477 | Regression_energy: 1.2103 | Regression_eta: 0.0642 | Regression_pt: 1.5614 | Regression_sin_phi: 0.0713 | LR: 1.58e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5130s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0031 | max_backward: 0.0398 | max_optimizer_step: 0.0657 | max_tensorboard_logging: 0.1202 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2801.0 | uss_MB: 1651.1 | child_rss_MB: 22650.7 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:15:37,251] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:15:37,331] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:16:34,541] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 07:16:34,541] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:17:29,483] INFO : Step 55000/500000 rank0 | Train Loss: 6.9628 | Train Loss Unweighted: 3.4042 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.0818 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1280 | Regression_eta: 0.0086 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3285 | Classification_binary: 3.0818 | Regression_cos_phi: 0.0985 | Regression_energy: 1.3877 | Regression_eta: 0.0686 | Regression_pt: 1.8624 | Regression_sin_phi: 0.1353 | LR: 1.62e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5262s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0030 | max_backward: 0.0405 | max_optimizer_step: 0.0692 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.3 | uss_MB: 1653.4 | child_rss_MB: 22649.8 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:17:37,945] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 07:17:37,945] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:18:40,779] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 07:18:40,779] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:19:21,700] INFO : Step 56000/500000 rank0 | Train Loss: 6.6129 | Train Loss Unweighted: 3.3263 | Train Loss Components: Classification: 0.0332 | Classification_binary: 3.0291 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1231 | Regression_eta: 0.0061 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3703 | Classification_binary: 3.0291 | Regression_cos_phi: 0.0597 | Regression_energy: 1.3341 | Regression_eta: 0.0484 | Regression_pt: 1.7137 | Regression_sin_phi: 0.0576 | LR: 1.67e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.5187s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0404 | max_loss: 0.0032 | max_backward: 0.0408 | max_optimizer_step: 0.0689 | max_tensorboard_logging: 0.1199 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.8 | uss_MB: 1653.9 | child_rss_MB: 22652.0 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:19:21,702] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:19:21,770] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:19:44,113] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 07:19:44,113] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:20:46,996] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 07:20:46,997] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:21:14,138] INFO : Step 57000/500000 rank0 | Train Loss: 7.2629 | Train Loss Unweighted: 3.3805 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.0306 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1467 | Regression_eta: 0.0053 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3354 | Classification_binary: 3.0306 | Regression_cos_phi: 0.1025 | Regression_energy: 1.5903 | Regression_eta: 0.0424 | Regression_pt: 2.0623 | Regression_sin_phi: 0.0994 | LR: 1.72e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5337s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0380 | max_loss: 0.0032 | max_backward: 0.0406 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.1310 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.9 | uss_MB: 1654.0 | child_rss_MB: 22650.5 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:21:50,659] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 07:21:50,659] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:22:54,844] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 07:22:54,844] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:23:08,191] INFO : Step 58000/500000 rank0 | Train Loss: 6.2871 | Train Loss Unweighted: 3.3968 | Train Loss Components: Classification: 0.0291 | Classification_binary: 3.1311 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1035 | Regression_eta: 0.0113 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3243 | Classification_binary: 3.1311 | Regression_cos_phi: 0.1069 | Regression_energy: 1.1220 | Regression_eta: 0.0898 | Regression_pt: 1.3859 | Regression_sin_phi: 0.1271 | LR: 1.76e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5748s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0031 | max_backward: 0.0405 | max_optimizer_step: 0.0643 | max_tensorboard_logging: 0.1685 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2800.5 | uss_MB: 1650.5 | child_rss_MB: 22652.5 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:23:08,193] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:23:08,265] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:23:58,236] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 07:23:58,236] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:25:00,239] INFO : Step 59000/500000 rank0 | Train Loss: 6.9724 | Train Loss Unweighted: 3.4296 | Train Loss Components: Classification: 0.0305 | Classification_binary: 3.1071 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1303 | Regression_eta: 0.0126 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3398 | Classification_binary: 3.1071 | Regression_cos_phi: 0.0779 | Regression_energy: 1.4128 | Regression_eta: 0.1006 | Regression_pt: 1.8367 | Regression_sin_phi: 0.0973 | LR: 1.81e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5538s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0029 | max_backward: 0.0402 | max_optimizer_step: 0.0644 | max_tensorboard_logging: 0.1523 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.3 | uss_MB: 1653.4 | child_rss_MB: 22645.8 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:25:01,339] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 07:25:01,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:26:04,263] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 07:26:04,263] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:26:52,902] INFO : Step 60000/500000 rank0 | Train Loss: 6.3340 | Train Loss Unweighted: 3.2435 | Train Loss Components: Classification: 0.0278 | Classification_binary: 2.9627 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1129 | Regression_eta: 0.0096 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3098 | Classification_binary: 2.9627 | Regression_cos_phi: 0.0745 | Regression_energy: 1.2235 | Regression_eta: 0.0764 | Regression_pt: 1.6038 | Regression_sin_phi: 0.0832 | LR: 1.86e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5424s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0380 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1500 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.4 | uss_MB: 1653.4 | child_rss_MB: 22621.9 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:26:52,904] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:26:52,972] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:26:52,973] INFO : Running validation on rank0 +rank0 [2026-11-14 07:26:52,973] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:26:52,973] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:26:53,125] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 07:26:53,246] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:26:53,364] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:26:53,433] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:26:53,500] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:26:53,569] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:26:53,638] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:26:53,707] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:26:53,808] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:26:53,925] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:26:54,041] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:26:54,111] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:26:54,178] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:26:54,245] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:26:54,312] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:26:54,381] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:26:54,482] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:26:54,489] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:26:54,491] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:26:54,494] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:26:54,496] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:26:54,498] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:26:54,500] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:26:54,502] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:26:54,504] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:26:54,506] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:26:54,508] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:26:54,510] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:26:54,512] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:26:54,514] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:26:54,516] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:26:54,519] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:26:56,019] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.3340 | Valid Loss=8.9484 | Train Loss Unweighted=3.2435 | Valid Loss Unweighted=3.8484 | Stale=1 | ETA=829.7m +rank0 [2026-11-14 07:26:56,019] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:26:56,420] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:26:56,429] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:26:56,483] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:26:57,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:26:57,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:26:57,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:26:57,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:26:58,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:26:58,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:26:58,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:26:58,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:26:59,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:26:59,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:26:59,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:26:59,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:27:00,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:27:00,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:27:00,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:27:00,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:27:00,929] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:27:00,937] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:01,522] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:01,913] INFO : Plotted number of elements +rank0 [2026-11-14 07:27:03,253] INFO : Plotted jets +rank0 [2026-11-14 07:27:04,236] INFO : Plotted jet ratio +rank0 [2026-11-14 07:27:19,667] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 07:27:19,667] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:28:22,371] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 07:28:22,371] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:28:56,780] INFO : Step 61000/500000 rank0 | Train Loss: 6.8925 | Train Loss Unweighted: 3.5547 | Train Loss Components: Classification: 0.0288 | Classification_binary: 3.2510 | Regression_cos_phi: 0.0133 | Regression_energy: 0.1184 | Regression_eta: 0.0102 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3209 | Classification_binary: 3.2510 | Regression_cos_phi: 0.1468 | Regression_energy: 1.2838 | Regression_eta: 0.0812 | Regression_pt: 1.7090 | Regression_sin_phi: 0.0998 | LR: 1.91e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5473s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0364 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0641 | max_tensorboard_logging: 0.1571 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2937.1 | uss_MB: 1787.5 | child_rss_MB: 22624.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:29:25,505] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 07:29:25,505] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:30:27,395] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 07:30:27,395] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:30:46,813] INFO : Step 62000/500000 rank0 | Train Loss: 6.2545 | Train Loss Unweighted: 3.3481 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.0831 | Regression_cos_phi: 0.0130 | Regression_energy: 0.1069 | Regression_eta: 0.0078 | Regression_pt: 0.1046 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2932 | Classification_binary: 3.0831 | Regression_cos_phi: 0.1427 | Regression_energy: 1.1593 | Regression_eta: 0.0623 | Regression_pt: 1.4457 | Regression_sin_phi: 0.0682 | LR: 1.95e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5465s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0364 | max_loss: 0.0027 | max_backward: 0.0415 | max_optimizer_step: 0.0623 | max_tensorboard_logging: 0.1967 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2828.1 | uss_MB: 1678.5 | child_rss_MB: 22607.9 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:30:46,815] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:30:46,880] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:31:30,741] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 07:31:30,742] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:32:33,201] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 07:32:33,201] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:32:39,639] INFO : Step 63000/500000 rank0 | Train Loss: 5.4502 | Train Loss Unweighted: 3.0795 | Train Loss Components: Classification: 0.0278 | Classification_binary: 2.8631 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0848 | Regression_eta: 0.0074 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3096 | Classification_binary: 2.8631 | Regression_cos_phi: 0.0759 | Regression_energy: 0.9192 | Regression_eta: 0.0592 | Regression_pt: 1.1721 | Regression_sin_phi: 0.0510 | LR: 2.00e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5259s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0028 | max_backward: 0.0412 | max_optimizer_step: 0.0654 | max_tensorboard_logging: 0.1568 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.4 | uss_MB: 1680.8 | child_rss_MB: 22602.2 | allocated_MB: 126.1 | reserved_MB: 658.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:33:35,270] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 07:33:35,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:34:31,007] INFO : Step 64000/500000 rank0 | Train Loss: 6.1490 | Train Loss Unweighted: 3.2676 | Train Loss Components: Classification: 0.0280 | Classification_binary: 3.0073 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1080 | Regression_eta: 0.0048 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3117 | Classification_binary: 3.0073 | Regression_cos_phi: 0.0646 | Regression_energy: 1.1712 | Regression_eta: 0.0384 | Regression_pt: 1.5036 | Regression_sin_phi: 0.0523 | LR: 2.05e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5124s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0026 | max_backward: 0.0424 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1352 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2830.2 | uss_MB: 1680.6 | child_rss_MB: 22609.9 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:34:31,009] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:34:31,080] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:34:38,699] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 07:34:38,699] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:35:41,507] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 07:35:41,507] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:36:24,144] INFO : Step 65000/500000 rank0 | Train Loss: 6.0762 | Train Loss Unweighted: 3.2428 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.9829 | Regression_cos_phi: 0.0134 | Regression_energy: 0.0996 | Regression_eta: 0.0120 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2441 | Classification_binary: 2.9829 | Regression_cos_phi: 0.1474 | Regression_energy: 1.0799 | Regression_eta: 0.0952 | Regression_pt: 1.4036 | Regression_sin_phi: 0.1232 | LR: 2.10e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5231s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0396 | max_loss: 0.0030 | max_backward: 0.0428 | max_optimizer_step: 0.0644 | max_tensorboard_logging: 0.1338 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2831.4 | uss_MB: 1681.7 | child_rss_MB: 22587.1 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:36:45,845] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 07:36:45,845] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:37:48,549] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 07:37:48,550] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:38:16,653] INFO : Step 66000/500000 rank0 | Train Loss: 5.9912 | Train Loss Unweighted: 3.1771 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.9208 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1047 | Regression_eta: 0.0082 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2617 | Classification_binary: 2.9208 | Regression_cos_phi: 0.0763 | Regression_energy: 1.1348 | Regression_eta: 0.0652 | Regression_pt: 1.4326 | Regression_sin_phi: 0.0999 | LR: 2.15e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5599s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0026 | max_backward: 0.0402 | max_optimizer_step: 0.0680 | max_tensorboard_logging: 0.1712 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2827.6 | uss_MB: 1677.9 | child_rss_MB: 22592.4 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 07:38:16,655] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:38:16,728] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:38:52,220] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 07:38:52,221] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:39:55,478] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 07:39:55,478] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:40:09,738] INFO : Step 67000/500000 rank0 | Train Loss: 6.1087 | Train Loss Unweighted: 3.1986 | Train Loss Components: Classification: 0.0250 | Classification_binary: 2.9284 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1037 | Regression_eta: 0.0226 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2786 | Classification_binary: 2.9284 | Regression_cos_phi: 0.0960 | Regression_energy: 1.1237 | Regression_eta: 0.1801 | Regression_pt: 1.4256 | Regression_sin_phi: 0.0765 | LR: 2.20e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5484s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.1479 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.6 | uss_MB: 1681.0 | child_rss_MB: 22599.5 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:40:59,609] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 07:40:59,609] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:42:02,758] INFO : Step 68000/500000 rank0 | Train Loss: 5.4546 | Train Loss Unweighted: 3.0139 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.7915 | Regression_cos_phi: 0.0075 | Regression_energy: 0.0912 | Regression_eta: 0.0068 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2540 | Classification_binary: 2.7915 | Regression_cos_phi: 0.0827 | Regression_energy: 0.9889 | Regression_eta: 0.0545 | Regression_pt: 1.2236 | Regression_sin_phi: 0.0593 | LR: 2.25e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5309s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0027 | max_backward: 0.0405 | max_optimizer_step: 0.0748 | max_tensorboard_logging: 0.1329 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2830.7 | uss_MB: 1681.0 | child_rss_MB: 22601.2 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:42:02,760] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:42:02,830] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:42:03,050] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 07:42:03,051] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:43:06,370] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 07:43:06,370] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:43:55,572] INFO : Step 69000/500000 rank0 | Train Loss: 6.1957 | Train Loss Unweighted: 3.2701 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.0011 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1064 | Regression_eta: 0.0104 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0146 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2918 | Classification_binary: 3.0011 | Regression_cos_phi: 0.1178 | Regression_energy: 1.1539 | Regression_eta: 0.0828 | Regression_pt: 1.3920 | Regression_sin_phi: 0.1564 | LR: 2.30e-04 | DataLoad Time: 0.0046s | Model Forward Time: 0.5017s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0364 | max_loss: 0.0027 | max_backward: 0.0406 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1335 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2830.8 | uss_MB: 1681.1 | child_rss_MB: 22606.9 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:44:09,727] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 07:44:09,727] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:45:12,720] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 07:45:12,720] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:45:48,563] INFO : Step 70000/500000 rank0 | Train Loss: 5.7326 | Train Loss Unweighted: 3.0332 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7872 | Regression_cos_phi: 0.0100 | Regression_energy: 0.0980 | Regression_eta: 0.0078 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2549 | Classification_binary: 2.7872 | Regression_cos_phi: 0.1099 | Regression_energy: 1.0622 | Regression_eta: 0.0624 | Regression_pt: 1.3599 | Regression_sin_phi: 0.0961 | LR: 2.35e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5334s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0408 | max_optimizer_step: 0.0801 | max_tensorboard_logging: 0.1430 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2827.7 | uss_MB: 1678.0 | child_rss_MB: 22612.2 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:45:48,565] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:45:48,636] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:45:48,636] INFO : Running validation on rank0 +rank0 [2026-11-14 07:45:48,636] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:45:48,636] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:45:48,781] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 07:45:48,906] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:45:49,026] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:45:49,096] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:45:49,163] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:45:49,231] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:45:49,298] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:45:49,366] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:45:49,466] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:45:49,582] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:45:49,698] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:45:49,766] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:45:49,833] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:45:49,898] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:45:49,965] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:45:50,032] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:45:50,141] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:45:50,148] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:45:50,150] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:45:50,152] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:45:50,154] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:45:50,156] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:45:50,159] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:45:50,161] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:45:50,163] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:45:50,165] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:45:50,167] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:45:50,169] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:45:50,171] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:45:50,173] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:45:50,175] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:45:50,176] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:45:51,674] INFO : VALIDATION | Step=70000/500000 | Train Loss=5.7326 | Valid Loss=9.5372 | Train Loss Unweighted=3.0332 | Valid Loss Unweighted=4.0008 | Stale=2 | ETA=811.3m +rank0 [2026-11-14 07:45:51,674] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:45:52,035] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:45:52,044] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:45:52,097] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:45:52,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:45:52,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:45:53,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:45:53,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:45:53,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:45:53,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:45:54,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:45:54,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:45:54,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:45:54,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:45:55,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:45:55,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:45:55,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:45:55,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:45:56,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:45:56,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:45:56,452] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:45:56,470] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:45:57,072] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:45:57,463] INFO : Plotted number of elements +rank0 [2026-11-14 07:45:58,807] INFO : Plotted jets +rank0 [2026-11-14 07:45:59,791] INFO : Plotted jet ratio +rank0 [2026-11-14 07:46:28,218] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 07:46:28,218] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:47:31,065] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 07:47:31,065] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:47:52,499] INFO : Step 71000/500000 rank0 | Train Loss: 5.4188 | Train Loss Unweighted: 2.9745 | Train Loss Components: Classification: 0.0288 | Classification_binary: 2.7531 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0871 | Regression_eta: 0.0039 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3202 | Classification_binary: 2.7531 | Regression_cos_phi: 0.0638 | Regression_energy: 0.9445 | Regression_eta: 0.0314 | Regression_pt: 1.2429 | Regression_sin_phi: 0.0628 | LR: 2.40e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5594s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0040 | max_backward: 0.0401 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1640 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2961.5 | uss_MB: 1812.1 | child_rss_MB: 22606.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:48:34,422] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 07:48:34,422] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:49:37,549] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 07:49:37,549] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:49:45,187] INFO : Step 72000/500000 rank0 | Train Loss: 5.0680 | Train Loss Unweighted: 2.8484 | Train Loss Components: Classification: 0.0253 | Classification_binary: 2.6421 | Regression_cos_phi: 0.0106 | Regression_energy: 0.0756 | Regression_eta: 0.0128 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2816 | Classification_binary: 2.6421 | Regression_cos_phi: 0.1162 | Regression_energy: 0.8200 | Regression_eta: 0.1020 | Regression_pt: 1.0118 | Regression_sin_phi: 0.0943 | LR: 2.45e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5843s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0030 | max_backward: 0.0452 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1883 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2961.5 | uss_MB: 1812.1 | child_rss_MB: 22610.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:49:45,190] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 07:49:45,259] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:50:41,452] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 07:50:41,452] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:51:38,208] INFO : Step 73000/500000 rank0 | Train Loss: 5.6168 | Train Loss Unweighted: 3.0654 | Train Loss Components: Classification: 0.0242 | Classification_binary: 2.8334 | Regression_cos_phi: 0.0074 | Regression_energy: 0.0944 | Regression_eta: 0.0065 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2696 | Classification_binary: 2.8334 | Regression_cos_phi: 0.0817 | Regression_energy: 1.0230 | Regression_eta: 0.0518 | Regression_pt: 1.2979 | Regression_sin_phi: 0.0594 | LR: 2.50e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5438s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0379 | max_loss: 0.0031 | max_backward: 0.0403 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1371 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2961.5 | uss_MB: 1812.1 | child_rss_MB: 22609.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:51:44,941] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 07:51:44,941] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:52:47,760] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 07:52:47,760] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:53:30,441] INFO : Step 74000/500000 rank0 | Train Loss: 5.9284 | Train Loss Unweighted: 3.0440 | Train Loss Components: Classification: 0.0265 | Classification_binary: 2.7805 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1033 | Regression_eta: 0.0103 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2951 | Classification_binary: 2.7805 | Regression_cos_phi: 0.0882 | Regression_energy: 1.1194 | Regression_eta: 0.0819 | Regression_pt: 1.4538 | Regression_sin_phi: 0.1095 | LR: 2.55e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5312s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0384 | max_loss: 0.0030 | max_backward: 0.0442 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1367 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2852.8 | uss_MB: 1703.4 | child_rss_MB: 22617.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:53:30,443] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 07:53:30,515] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:53:51,104] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 07:53:51,104] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:54:53,115] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 07:54:53,115] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:55:21,860] INFO : Step 75000/500000 rank0 | Train Loss: 4.8602 | Train Loss Unweighted: 2.8178 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.6287 | Regression_cos_phi: 0.0093 | Regression_energy: 0.0714 | Regression_eta: 0.0094 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2545 | Classification_binary: 2.6287 | Regression_cos_phi: 0.1025 | Regression_energy: 0.7735 | Regression_eta: 0.0747 | Regression_pt: 0.9324 | Regression_sin_phi: 0.0938 | LR: 2.60e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5828s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0381 | max_loss: 0.0031 | max_backward: 0.0427 | max_optimizer_step: 0.0748 | max_tensorboard_logging: 0.1507 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2855.5 | uss_MB: 1706.1 | child_rss_MB: 22622.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:55:57,259] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 07:55:57,260] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:56:59,897] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 07:56:59,897] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:57:14,437] INFO : Step 76000/500000 rank0 | Train Loss: 5.0480 | Train Loss Unweighted: 2.8762 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.6785 | Regression_cos_phi: 0.0074 | Regression_energy: 0.0792 | Regression_eta: 0.0049 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2511 | Classification_binary: 2.6785 | Regression_cos_phi: 0.0810 | Regression_energy: 0.8586 | Regression_eta: 0.0393 | Regression_pt: 1.0804 | Regression_sin_phi: 0.0590 | LR: 2.65e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5125s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0035 | max_backward: 0.0400 | max_optimizer_step: 0.0616 | max_tensorboard_logging: 0.1364 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2855.5 | uss_MB: 1706.1 | child_rss_MB: 22627.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:57:14,439] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 07:57:14,506] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:58:03,163] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 07:58:03,163] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:59:06,518] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 07:59:06,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:59:07,634] INFO : Step 77000/500000 rank0 | Train Loss: 5.0269 | Train Loss Unweighted: 2.6675 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4533 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0871 | Regression_eta: 0.0059 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2336 | Classification_binary: 2.4533 | Regression_cos_phi: 0.0740 | Regression_energy: 0.9439 | Regression_eta: 0.0470 | Regression_pt: 1.2111 | Regression_sin_phi: 0.0640 | LR: 2.70e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5391s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0031 | max_backward: 0.0413 | max_optimizer_step: 0.0763 | max_tensorboard_logging: 0.1400 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.8 | uss_MB: 1706.4 | child_rss_MB: 22623.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 08:00:10,616] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 08:00:10,616] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:01:00,932] INFO : Step 78000/500000 rank0 | Train Loss: 4.6779 | Train Loss Unweighted: 2.6955 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.5148 | Regression_cos_phi: 0.0098 | Regression_energy: 0.0699 | Regression_eta: 0.0039 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2285 | Classification_binary: 2.5148 | Regression_cos_phi: 0.1081 | Regression_energy: 0.7572 | Regression_eta: 0.0313 | Regression_pt: 0.9677 | Regression_sin_phi: 0.0702 | LR: 2.75e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5134s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0026 | max_backward: 0.0389 | max_optimizer_step: 0.0631 | max_tensorboard_logging: 0.1368 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.8 | uss_MB: 1706.4 | child_rss_MB: 22631.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:01:00,934] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 08:01:01,004] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 08:01:14,230] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 08:01:14,230] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:17,304] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 08:02:17,304] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:53,391] INFO : Step 79000/500000 rank0 | Train Loss: 5.0982 | Train Loss Unweighted: 2.8918 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6902 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0791 | Regression_eta: 0.0083 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2162 | Classification_binary: 2.6902 | Regression_cos_phi: 0.0738 | Regression_energy: 0.8569 | Regression_eta: 0.0664 | Regression_pt: 1.1110 | Regression_sin_phi: 0.0838 | LR: 2.80e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5699s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0031 | max_backward: 0.0406 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1844 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.4 | uss_MB: 1706.0 | child_rss_MB: 22621.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:03:20,370] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 08:03:20,370] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:04:23,295] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 08:04:23,295] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:04:46,068] INFO : Step 80000/500000 rank0 | Train Loss: 4.2202 | Train Loss Unweighted: 2.4521 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.2897 | Regression_cos_phi: 0.0089 | Regression_energy: 0.0618 | Regression_eta: 0.0053 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2369 | Classification_binary: 2.2897 | Regression_cos_phi: 0.0985 | Regression_energy: 0.6704 | Regression_eta: 0.0424 | Regression_pt: 0.8241 | Regression_sin_phi: 0.0583 | LR: 2.85e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5206s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1269 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.5 | uss_MB: 1706.1 | child_rss_MB: 22626.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:04:46,070] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 08:04:46,139] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 08:04:46,139] INFO : Running validation on rank0 +rank0 [2026-11-14 08:04:46,139] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 08:04:46,139] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 08:04:46,288] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 08:04:46,412] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 08:04:46,531] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 08:04:46,601] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 08:04:46,669] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 08:04:46,735] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 08:04:46,804] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 08:04:46,873] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 08:04:46,978] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 08:04:47,097] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 08:04:47,217] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 08:04:47,285] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 08:04:47,352] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 08:04:47,421] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 08:04:47,489] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 08:04:47,559] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 08:04:47,662] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 08:04:47,669] INFO : plotted pt_target histogram +rank0 [2026-11-14 08:04:47,671] INFO : plotted pt_pred histogram +rank0 [2026-11-14 08:04:47,673] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 08:04:47,675] INFO : plotted eta_target histogram +rank0 [2026-11-14 08:04:47,677] INFO : plotted eta_pred histogram +rank0 [2026-11-14 08:04:47,679] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 08:04:47,681] INFO : plotted sphi_target histogram +rank0 [2026-11-14 08:04:47,683] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 08:04:47,685] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 08:04:47,687] INFO : plotted cphi_target histogram +rank0 [2026-11-14 08:04:47,689] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 08:04:47,691] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 08:04:47,692] INFO : plotted energy_target histogram +rank0 [2026-11-14 08:04:47,694] INFO : plotted energy_pred histogram +rank0 [2026-11-14 08:04:47,697] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 08:04:49,196] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.2202 | Valid Loss=10.4248 | Train Loss Unweighted=2.4521 | Valid Loss Unweighted=4.4044 | Stale=3 | ETA=792.9m +rank0 [2026-11-14 08:04:49,197] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 08:04:49,700] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 08:04:49,707] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 08:04:49,761] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 08:04:50,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 08:04:50,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 08:04:50,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 08:04:51,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 08:04:51,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 08:04:51,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 08:04:51,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 08:04:52,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 08:04:52,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 08:04:52,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 08:04:52,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 08:04:53,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 08:04:53,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 08:04:53,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 08:04:53,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 08:04:54,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 08:04:54,111] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 08:04:54,118] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:04:54,702] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:04:55,101] INFO : Plotted number of elements +rank0 [2026-11-14 08:04:56,450] INFO : Plotted jets +rank0 [2026-11-14 08:04:57,431] INFO : Plotted jet ratio +rank0 [2026-11-14 08:04:57,898] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 08:04:57,898] INFO : Training completed. Total time on device 0: 151.175min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..79ab706230963a65256f0ffa3601655e8bea2512 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/train.log.0 @@ -0,0 +1,1351 @@ +rank0 [2026-11-14 05:33:45,302] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 05:33:45,303] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 05:33:45,304] INFO : Instantiating model +rank0 [2026-11-14 05:33:45,327] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 05:33:45,328] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 05:33:45,329] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 05:33:45,329] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 05:33:45,329] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 05:33:45,330] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 05:33:45,330] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,332] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 05:33:45,332] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,335] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 05:33:45,335] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,338] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 05:33:45,338] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,341] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 05:33:45,341] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,344] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 05:33:45,344] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,346] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 05:33:45,347] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 05:33:45,347] INFO : Initializing output DNNs +rank0 [2026-11-14 05:33:45,347] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,350] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,357] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 05:33:45,358] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 05:33:45,358] INFO : jagged_attention enabled=True +rank0 [2026-11-14 05:33:45,358] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 05:33:45,358] INFO : task_queries enabled=True +rank0 [2026-11-14 05:33:45,358] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 05:33:45,358] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 05:33:45,358] INFO : classification_query parameters: 256 +rank0 [2026-11-14 05:33:45,358] INFO : regression_query parameters: 256 +rank0 [2026-11-14 05:33:45,358] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 05:33:45,358] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 05:33:45,358] INFO : output_mode=elementwise +rank0 [2026-11-14 05:33:45,358] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 05:33:45,358] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 05:33:45,358] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 05:33:45,358] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 05:33:45,358] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 05:33:45,358] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 05:33:45,358] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 05:33:45,358] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 05:33:45,359] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 05:33:45,360] INFO : MLPF __init__ done +rank0 [2026-11-14 05:33:45,360] INFO : Instantiated model +rank0 [2026-11-14 05:33:45,360] INFO : Moving model to device rank=0 +rank0 [2026-11-14 05:33:45,372] INFO : Moved model to device rank=0 +rank0 [2026-11-14 05:33:45,373] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 05:33:45,471] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 05:33:45,471] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 05:33:45,472] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 05:33:45,472] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 05:33:45,472] INFO : Total parameters: 3910930 +rank0 [2026-11-14 05:33:45,475] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 05:33:45,479] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794 +rank0 [2026-11-14 05:33:45,479] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794 +rank0 [2026-11-14 05:33:45,479] INFO : Getting dataloaders +rank0 [2026-11-14 05:33:45,479] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:33:46,703] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,756] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,819] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,869] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,897] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,936] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,982] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:47,038] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:47,081] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:47,122] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:47,122] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 05:33:47,122] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 05:33:47,122] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 05:33:47,122] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:33:47,131] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,140] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,148] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,158] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,166] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,173] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,181] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,188] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,197] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,206] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,206] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 05:33:47,207] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 05:33:47,250] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:33:47,399] INFO : Got dataloaders +rank0 [2026-11-14 05:33:47,399] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 05:33:47,399] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 05:33:47,400] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 05:33:47,406] INFO : Creating train_iterator +rank0 [2026-11-14 05:33:47,406] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:33:47,433] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:33:47,588] INFO : Created train_iterator +rank0 [2026-11-14 05:34:00,873] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.12873676776886, 'Classification': 0.6399854928255081, 'Regression_pt': 0.5156782293319702, 'Regression_eta': 0.8953252410888672, 'Regression_sin_phi': 0.6647196558117866, 'Regression_cos_phi': 0.6473717069625855, 'Regression_energy': 0.657646809220314}, weights={'Classification_binary': 1.0, 'Classification': 11.138903617858887, 'Regression_pt': 13.824002265930176, 'Regression_eta': 7.962175369262695, 'Regression_sin_phi': 10.72442626953125, 'Regression_cos_phi': 11.01181411743164, 'Regression_energy': 10.839764595031738} +rank0 [2026-11-14 05:34:53,616] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 05:34:53,616] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:43,505] INFO : Step 1000/500000 rank0 | Train Loss: 11.6596 | Train Loss Unweighted: 5.1031 | Train Loss Components: Classification: 0.0671 | Classification_binary: 4.5029 | Regression_cos_phi: 0.0193 | Regression_energy: 0.2242 | Regression_eta: 0.0294 | Regression_pt: 0.2394 | Regression_sin_phi: 0.0208 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.7477 | Classification_binary: 4.5029 | Regression_cos_phi: 0.2126 | Regression_energy: 2.4305 | Regression_eta: 0.2339 | Regression_pt: 3.3089 | Regression_sin_phi: 0.2231 | LR: 2.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5817s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0388 | max_loss: 0.0029 | max_backward: 0.0491 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1917 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2578.7 | uss_MB: 1412.7 | child_rss_MB: 22594.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:35:57,450] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 05:35:57,450] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:00,522] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 05:37:00,522] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:36,092] INFO : Step 2000/500000 rank0 | Train Loss: 10.5215 | Train Loss Unweighted: 4.7396 | Train Loss Components: Classification: 0.0483 | Classification_binary: 4.2137 | Regression_cos_phi: 0.0150 | Regression_energy: 0.2091 | Regression_eta: 0.0169 | Regression_pt: 0.2150 | Regression_sin_phi: 0.0215 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.5385 | Classification_binary: 4.2137 | Regression_cos_phi: 0.1655 | Regression_energy: 2.2670 | Regression_eta: 0.1344 | Regression_pt: 2.9717 | Regression_sin_phi: 0.2306 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5222s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0025 | max_backward: 0.0413 | max_optimizer_step: 0.0626 | max_tensorboard_logging: 0.1563 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2579.5 | uss_MB: 1413.6 | child_rss_MB: 22606.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 42.3 +rank0 [2026-11-14 05:37:36,094] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:38:03,688] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 05:38:03,688] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:06,487] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 05:39:06,487] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:28,333] INFO : Step 3000/500000 rank0 | Train Loss: 10.6797 | Train Loss Unweighted: 4.5679 | Train Loss Components: Classification: 0.0419 | Classification_binary: 4.0139 | Regression_cos_phi: 0.0129 | Regression_energy: 0.2329 | Regression_eta: 0.0176 | Regression_pt: 0.2338 | Regression_sin_phi: 0.0149 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4663 | Classification_binary: 4.0139 | Regression_cos_phi: 0.1419 | Regression_energy: 2.5249 | Regression_eta: 0.1400 | Regression_pt: 3.2325 | Regression_sin_phi: 0.1603 | LR: 2.05e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.6454s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.2768 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2582.4 | uss_MB: 1416.8 | child_rss_MB: 22610.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 16.3 +rank0 [2026-11-14 05:40:10,220] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 05:40:10,220] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:13,215] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 05:41:13,215] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:21,015] INFO : Step 4000/500000 rank0 | Train Loss: 9.3290 | Train Loss Unweighted: 4.3481 | Train Loss Components: Classification: 0.0463 | Classification_binary: 3.8951 | Regression_cos_phi: 0.0167 | Regression_energy: 0.1852 | Regression_eta: 0.0122 | Regression_pt: 0.1819 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.5156 | Classification_binary: 3.8951 | Regression_cos_phi: 0.1835 | Regression_energy: 2.0074 | Regression_eta: 0.0974 | Regression_pt: 2.5144 | Regression_sin_phi: 0.1154 | LR: 2.08e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5099s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0027 | max_backward: 0.0400 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1275 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2582.6 | uss_MB: 1417.0 | child_rss_MB: 22610.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:41:21,017] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:42:16,228] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 05:42:16,228] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:43:12,674] INFO : Step 5000/500000 rank0 | Train Loss: 11.4198 | Train Loss Unweighted: 4.7503 | Train Loss Components: Classification: 0.0443 | Classification_binary: 4.1483 | Regression_cos_phi: 0.0163 | Regression_energy: 0.2563 | Regression_eta: 0.0146 | Regression_pt: 0.2591 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4936 | Classification_binary: 4.1483 | Regression_cos_phi: 0.1795 | Regression_energy: 2.7779 | Regression_eta: 0.1159 | Regression_pt: 3.5816 | Regression_sin_phi: 0.1230 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5594s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0350 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1840 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2581.7 | uss_MB: 1416.2 | child_rss_MB: 22611.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:43:19,074] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 05:43:19,075] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:21,206] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 05:44:21,206] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:45:03,640] INFO : Step 6000/500000 rank0 | Train Loss: 8.9600 | Train Loss Unweighted: 4.3570 | Train Loss Components: Classification: 0.0434 | Classification_binary: 3.9415 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1661 | Regression_eta: 0.0096 | Regression_pt: 0.1772 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4832 | Classification_binary: 3.9415 | Regression_cos_phi: 0.0875 | Regression_energy: 1.8010 | Regression_eta: 0.0762 | Regression_pt: 2.4501 | Regression_sin_phi: 0.1206 | LR: 2.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4868s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0345 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0746 | max_tensorboard_logging: 0.1284 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2581.8 | uss_MB: 1416.2 | child_rss_MB: 22608.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:45:03,642] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:45:23,811] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 05:45:23,811] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:25,842] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 05:46:25,842] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:54,719] INFO : Step 7000/500000 rank0 | Train Loss: 9.7732 | Train Loss Unweighted: 4.2527 | Train Loss Components: Classification: 0.0470 | Classification_binary: 3.7504 | Regression_cos_phi: 0.0165 | Regression_energy: 0.2044 | Regression_eta: 0.0176 | Regression_pt: 0.2054 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.5230 | Classification_binary: 3.7504 | Regression_cos_phi: 0.1820 | Regression_energy: 2.2159 | Regression_eta: 0.1400 | Regression_pt: 2.8393 | Regression_sin_phi: 0.1227 | LR: 2.26e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5028s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0377 | max_loss: 0.0029 | max_backward: 0.0416 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2583.6 | uss_MB: 1418.1 | child_rss_MB: 22613.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:47:29,122] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 05:47:29,122] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:31,994] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 05:48:31,994] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:47,085] INFO : Step 8000/500000 rank0 | Train Loss: 9.3288 | Train Loss Unweighted: 4.3855 | Train Loss Components: Classification: 0.0356 | Classification_binary: 3.9366 | Regression_cos_phi: 0.0151 | Regression_energy: 0.1869 | Regression_eta: 0.0140 | Regression_pt: 0.1858 | Regression_sin_phi: 0.0116 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3963 | Classification_binary: 3.9366 | Regression_cos_phi: 0.1659 | Regression_energy: 2.0256 | Regression_eta: 0.1115 | Regression_pt: 2.5685 | Regression_sin_phi: 0.1245 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5648s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0386 | max_loss: 0.0028 | max_backward: 0.0414 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1818 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2583.8 | uss_MB: 1418.2 | child_rss_MB: 22611.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:48:47,087] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:48:47,156] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:49:34,692] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 05:49:34,692] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:37,071] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 05:50:37,071] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:38,370] INFO : Step 9000/500000 rank0 | Train Loss: 10.4118 | Train Loss Unweighted: 4.4534 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.9188 | Regression_cos_phi: 0.0114 | Regression_energy: 0.2330 | Regression_eta: 0.0090 | Regression_pt: 0.2393 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3348 | Classification_binary: 3.9188 | Regression_cos_phi: 0.1257 | Regression_energy: 2.5254 | Regression_eta: 0.0714 | Regression_pt: 3.3080 | Regression_sin_phi: 0.1277 | LR: 2.43e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4931s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0384 | max_loss: 0.0031 | max_backward: 0.0419 | max_optimizer_step: 0.0620 | max_tensorboard_logging: 0.1228 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2581.4 | uss_MB: 1415.8 | child_rss_MB: 22611.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:51:40,068] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 05:51:40,068] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:30,410] INFO : Step 10000/500000 rank0 | Train Loss: 9.6583 | Train Loss Unweighted: 4.5131 | Train Loss Components: Classification: 0.0349 | Classification_binary: 4.0479 | Regression_cos_phi: 0.0140 | Regression_energy: 0.2010 | Regression_eta: 0.0109 | Regression_pt: 0.1968 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3889 | Classification_binary: 4.0479 | Regression_cos_phi: 0.1543 | Regression_energy: 2.1783 | Regression_eta: 0.0865 | Regression_pt: 2.7205 | Regression_sin_phi: 0.0820 | LR: 2.52e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5027s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0034 | max_backward: 0.0406 | max_optimizer_step: 0.0637 | max_tensorboard_logging: 0.1245 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2581.8 | uss_MB: 1416.2 | child_rss_MB: 22610.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:52:30,411] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:52:30,478] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:52:30,479] INFO : Running validation on rank0 +rank0 [2026-11-14 05:52:30,479] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:52:30,479] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:52:30,684] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 05:52:30,882] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:52:30,965] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:52:31,021] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:52:31,079] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:52:31,129] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:52:31,186] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:52:31,242] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:52:31,292] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:52:31,373] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:52:31,453] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:52:31,507] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:52:31,562] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:52:31,612] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:52:31,666] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:52:31,724] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:52:31,774] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:52:31,778] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:52:31,780] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:52:31,783] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:52:31,785] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:52:31,787] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:52:31,789] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:52:31,791] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:52:31,793] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:52:31,795] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:52:31,797] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:52:31,799] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:52:31,801] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:52:31,803] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:52:31,805] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:52:31,807] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:52:33,386] INFO : VALIDATION | Step=10000/500000 | Train Loss=9.6583 | Valid Loss=9.9378 | Train Loss Unweighted=4.5131 | Valid Loss Unweighted=4.4089 | Stale=0 | ETA=919.6m +rank0 [2026-11-14 05:52:33,387] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:52:33,836] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:52:33,844] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:33,898] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:52:34,610] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:52:34,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:52:35,128] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:52:35,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:52:35,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:52:35,887] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:52:36,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:52:36,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:52:36,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:52:36,925] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:52:37,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:52:37,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:52:37,691] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:52:37,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:52:38,201] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:52:38,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:52:38,497] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:52:38,505] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:39,095] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:39,858] INFO : Plotted number of elements +rank0 [2026-11-14 05:52:41,251] INFO : Plotted jets +rank0 [2026-11-14 05:52:42,238] INFO : Plotted jet ratio +rank0 [2026-11-14 05:52:55,581] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 05:52:55,582] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:53:57,279] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 05:53:57,279] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:34,425] INFO : Step 11000/500000 rank0 | Train Loss: 9.8952 | Train Loss Unweighted: 4.4012 | Train Loss Components: Classification: 0.0358 | Classification_binary: 3.9067 | Regression_cos_phi: 0.0073 | Regression_energy: 0.2072 | Regression_eta: 0.0117 | Regression_pt: 0.2185 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3990 | Classification_binary: 3.9067 | Regression_cos_phi: 0.0800 | Regression_energy: 2.2463 | Regression_eta: 0.0932 | Regression_pt: 3.0209 | Regression_sin_phi: 0.1491 | LR: 2.63e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5468s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0386 | max_loss: 0.0032 | max_backward: 0.0411 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1333 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2764.3 | uss_MB: 1613.5 | child_rss_MB: 22625.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:55:00,349] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 05:55:00,349] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:01,065] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 05:56:01,066] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:23,723] INFO : Step 12000/500000 rank0 | Train Loss: 8.4180 | Train Loss Unweighted: 3.9754 | Train Loss Components: Classification: 0.0341 | Classification_binary: 3.5708 | Regression_cos_phi: 0.0179 | Regression_energy: 0.1598 | Regression_eta: 0.0166 | Regression_pt: 0.1664 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3798 | Classification_binary: 3.5708 | Regression_cos_phi: 0.1974 | Regression_energy: 1.7322 | Regression_eta: 0.1321 | Regression_pt: 2.3009 | Regression_sin_phi: 0.1048 | LR: 2.75e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5349s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1524 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2764.4 | uss_MB: 1613.5 | child_rss_MB: 22622.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:56:23,725] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:56:23,796] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:57:04,065] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 05:57:04,065] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:06,559] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 05:58:06,559] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:15,174] INFO : Step 13000/500000 rank0 | Train Loss: 8.3447 | Train Loss Unweighted: 3.8651 | Train Loss Components: Classification: 0.0274 | Classification_binary: 3.4625 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1676 | Regression_eta: 0.0090 | Regression_pt: 0.1796 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3058 | Classification_binary: 3.4625 | Regression_cos_phi: 0.0890 | Regression_energy: 1.8172 | Regression_eta: 0.0717 | Regression_pt: 2.4828 | Regression_sin_phi: 0.1157 | LR: 2.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5045s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0401 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1290 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2693.3 | uss_MB: 1542.5 | child_rss_MB: 22626.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:59:09,470] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:59:09,470] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:06,846] INFO : Step 14000/500000 rank0 | Train Loss: 7.7032 | Train Loss Unweighted: 3.8688 | Train Loss Components: Classification: 0.0383 | Classification_binary: 3.5226 | Regression_cos_phi: 0.0096 | Regression_energy: 0.1417 | Regression_eta: 0.0055 | Regression_pt: 0.1445 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4267 | Classification_binary: 3.5226 | Regression_cos_phi: 0.1060 | Regression_energy: 1.5355 | Regression_eta: 0.0439 | Regression_pt: 1.9978 | Regression_sin_phi: 0.0707 | LR: 3.02e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5553s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0029 | max_backward: 0.0399 | max_optimizer_step: 0.0768 | max_tensorboard_logging: 0.1569 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2693.6 | uss_MB: 1542.8 | child_rss_MB: 22624.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:00:06,848] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:00:06,920] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 06:00:12,479] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 06:00:12,479] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:15,424] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 06:01:15,424] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:59,742] INFO : Step 15000/500000 rank0 | Train Loss: 8.3672 | Train Loss Unweighted: 4.1495 | Train Loss Components: Classification: 0.0321 | Classification_binary: 3.7667 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1615 | Regression_eta: 0.0120 | Regression_pt: 0.1590 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3578 | Classification_binary: 3.7667 | Regression_cos_phi: 0.1269 | Regression_energy: 1.7504 | Regression_eta: 0.0954 | Regression_pt: 2.1981 | Regression_sin_phi: 0.0720 | LR: 3.17e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5511s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0389 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1602 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2696.3 | uss_MB: 1545.5 | child_rss_MB: 22625.4 | allocated_MB: 126.1 | reserved_MB: 650.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:02:19,183] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 06:02:19,183] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:21,441] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 06:03:21,441] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:51,077] INFO : Step 16000/500000 rank0 | Train Loss: 8.2910 | Train Loss Unweighted: 4.0729 | Train Loss Components: Classification: 0.0317 | Classification_binary: 3.6916 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1634 | Regression_eta: 0.0096 | Regression_pt: 0.1619 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3533 | Classification_binary: 3.6916 | Regression_cos_phi: 0.0912 | Regression_energy: 1.7716 | Regression_eta: 0.0760 | Regression_pt: 2.2384 | Regression_sin_phi: 0.0688 | LR: 3.33e-05 | DataLoad Time: 0.0051s | Model Forward Time: 0.5303s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0382 | max_loss: 0.0029 | max_backward: 0.0431 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1532 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2696.3 | uss_MB: 1545.4 | child_rss_MB: 22628.0 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:03:51,078] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:03:51,151] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 06:04:24,287] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 06:04:24,287] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:26,712] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 06:05:26,712] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:42,723] INFO : Step 17000/500000 rank0 | Train Loss: 8.8821 | Train Loss Unweighted: 4.0547 | Train Loss Components: Classification: 0.0384 | Classification_binary: 3.6183 | Regression_cos_phi: 0.0203 | Regression_energy: 0.1804 | Regression_eta: 0.0068 | Regression_pt: 0.1806 | Regression_sin_phi: 0.0100 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4280 | Classification_binary: 3.6183 | Regression_cos_phi: 0.2234 | Regression_energy: 1.9550 | Regression_eta: 0.0539 | Regression_pt: 2.4962 | Regression_sin_phi: 0.1074 | LR: 3.51e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5111s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0691 | max_tensorboard_logging: 0.1235 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2692.9 | uss_MB: 1542.0 | child_rss_MB: 22628.1 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:30,184] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 06:06:30,184] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:32,227] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 06:07:32,227] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:34,426] INFO : Step 18000/500000 rank0 | Train Loss: 9.1418 | Train Loss Unweighted: 4.2839 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.8441 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1889 | Regression_eta: 0.0119 | Regression_pt: 0.1855 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3774 | Classification_binary: 3.8441 | Regression_cos_phi: 0.1143 | Regression_energy: 2.0474 | Regression_eta: 0.0949 | Regression_pt: 2.5645 | Regression_sin_phi: 0.0990 | LR: 3.69e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.5296s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0026 | max_backward: 0.0407 | max_optimizer_step: 0.0650 | max_tensorboard_logging: 0.1525 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2693.7 | uss_MB: 1542.9 | child_rss_MB: 22635.6 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 06:07:34,428] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:07:34,498] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 06:08:35,008] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 06:08:35,008] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:26,167] INFO : Step 19000/500000 rank0 | Train Loss: 10.5853 | Train Loss Unweighted: 4.2671 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.6971 | Regression_cos_phi: 0.0100 | Regression_energy: 0.2471 | Regression_eta: 0.0148 | Regression_pt: 0.2486 | Regression_sin_phi: 0.0156 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3774 | Classification_binary: 3.6971 | Regression_cos_phi: 0.1099 | Regression_energy: 2.6790 | Regression_eta: 0.1182 | Regression_pt: 3.4369 | Regression_sin_phi: 0.1668 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5558s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0028 | max_backward: 0.0392 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1737 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2696.2 | uss_MB: 1545.4 | child_rss_MB: 22633.8 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 06:09:38,103] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 06:09:38,103] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:10:40,925] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 06:10:40,925] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:18,831] INFO : Step 20000/500000 rank0 | Train Loss: 9.0492 | Train Loss Unweighted: 4.1925 | Train Loss Components: Classification: 0.0308 | Classification_binary: 3.7550 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1878 | Regression_eta: 0.0083 | Regression_pt: 0.1897 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3427 | Classification_binary: 3.7550 | Regression_cos_phi: 0.1187 | Regression_energy: 2.0355 | Regression_eta: 0.0660 | Regression_pt: 2.6227 | Regression_sin_phi: 0.1085 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5041s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0028 | max_backward: 0.0418 | max_optimizer_step: 0.0638 | max_tensorboard_logging: 0.1216 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2696.3 | uss_MB: 1545.5 | child_rss_MB: 22630.0 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:11:18,833] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:11:18,904] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:11:18,905] INFO : Running validation on rank0 +rank0 [2026-11-14 06:11:18,905] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:11:18,905] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:11:19,055] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 06:11:19,180] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:11:19,301] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:11:19,372] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:11:19,441] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:11:19,509] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:11:19,578] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:11:19,648] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:11:19,749] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:11:19,869] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:11:19,989] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:11:20,059] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:11:20,129] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:11:20,198] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:11:20,268] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:11:20,337] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:11:20,449] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:11:20,456] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:11:20,458] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:11:20,460] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:11:20,462] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:11:20,464] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:11:20,467] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:11:20,469] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:11:20,471] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:11:20,473] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:11:20,475] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:11:20,477] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:11:20,479] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:11:20,481] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:11:20,483] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:11:20,485] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:11:22,117] INFO : VALIDATION | Step=20000/500000 | Train Loss=9.0492 | Valid Loss=9.5361 | Train Loss Unweighted=4.1925 | Valid Loss Unweighted=4.2354 | Stale=0 | ETA=901.9m +rank0 [2026-11-14 06:11:22,131] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:11:22,501] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:11:22,536] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:22,587] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:11:23,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:11:23,374] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:11:23,633] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:11:23,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:11:24,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:11:24,412] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:11:24,667] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:11:24,924] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:11:25,185] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:11:25,441] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:11:25,703] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:11:25,957] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:11:27,482] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:11:27,732] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:11:27,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:11:28,225] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:11:28,282] INFO : Time taken to make predictions on device 0 is: 0.09 min +rank0 [2026-11-14 06:11:28,290] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:28,888] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:29,293] INFO : Plotted number of elements +rank0 [2026-11-14 06:11:30,645] INFO : Plotted jets +rank0 [2026-11-14 06:11:31,649] INFO : Plotted jet ratio +rank0 [2026-11-14 06:11:58,086] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 06:11:58,086] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:01,031] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 06:13:01,031] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:24,597] INFO : Step 21000/500000 rank0 | Train Loss: 9.7959 | Train Loss Unweighted: 4.4097 | Train Loss Components: Classification: 0.0235 | Classification_binary: 3.9260 | Regression_cos_phi: 0.0079 | Regression_energy: 0.2085 | Regression_eta: 0.0134 | Regression_pt: 0.2204 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2622 | Classification_binary: 3.9260 | Regression_cos_phi: 0.0871 | Regression_energy: 2.2597 | Regression_eta: 0.1066 | Regression_pt: 3.0463 | Regression_sin_phi: 0.1081 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5436s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0031 | max_backward: 0.0414 | max_optimizer_step: 0.0724 | max_tensorboard_logging: 0.1440 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2726.7 | uss_MB: 1576.3 | child_rss_MB: 22630.2 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:14:04,381] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 06:14:04,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:07,378] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 06:15:07,378] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:16,985] INFO : Step 22000/500000 rank0 | Train Loss: 8.6585 | Train Loss Unweighted: 4.1771 | Train Loss Components: Classification: 0.0312 | Classification_binary: 3.7726 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1741 | Regression_eta: 0.0072 | Regression_pt: 0.1720 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3471 | Classification_binary: 3.7726 | Regression_cos_phi: 0.0777 | Regression_energy: 1.8870 | Regression_eta: 0.0575 | Regression_pt: 2.3778 | Regression_sin_phi: 0.1388 | LR: 4.50e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5249s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0382 | max_loss: 0.0029 | max_backward: 0.0431 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1318 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2727.6 | uss_MB: 1577.3 | child_rss_MB: 22630.6 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:15:16,987] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:15:17,061] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:16:10,833] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 06:16:10,833] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:09,549] INFO : Step 23000/500000 rank0 | Train Loss: 7.5217 | Train Loss Unweighted: 3.7963 | Train Loss Components: Classification: 0.0289 | Classification_binary: 3.4619 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1426 | Regression_eta: 0.0046 | Regression_pt: 0.1475 | Regression_sin_phi: 0.0058 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3214 | Classification_binary: 3.4619 | Regression_cos_phi: 0.0545 | Regression_energy: 1.5459 | Regression_eta: 0.0367 | Regression_pt: 2.0388 | Regression_sin_phi: 0.0625 | LR: 4.73e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5229s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0031 | max_backward: 0.0418 | max_optimizer_step: 0.0705 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.6 | uss_MB: 1580.3 | child_rss_MB: 22643.6 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:17:14,245] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 06:17:14,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:16,495] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 06:18:16,495] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:19:02,043] INFO : Step 24000/500000 rank0 | Train Loss: 7.5216 | Train Loss Unweighted: 3.9589 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.6326 | Regression_cos_phi: 0.0112 | Regression_energy: 0.1312 | Regression_eta: 0.0130 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0142 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3305 | Classification_binary: 3.6326 | Regression_cos_phi: 0.1229 | Regression_energy: 1.4221 | Regression_eta: 0.1036 | Regression_pt: 1.7575 | Regression_sin_phi: 0.1524 | LR: 4.97e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4692s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0026 | max_backward: 0.0395 | max_optimizer_step: 0.0608 | max_tensorboard_logging: 0.1278 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.7 | uss_MB: 1580.3 | child_rss_MB: 22641.2 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:19:02,045] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:19:02,114] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:19:20,143] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 06:19:20,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:21,608] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 06:20:21,608] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:53,021] INFO : Step 25000/500000 rank0 | Train Loss: 8.1193 | Train Loss Unweighted: 3.8959 | Train Loss Components: Classification: 0.0344 | Classification_binary: 3.5117 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1620 | Regression_eta: 0.0132 | Regression_pt: 0.1573 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3835 | Classification_binary: 3.5117 | Regression_cos_phi: 0.1156 | Regression_energy: 1.7564 | Regression_eta: 0.1050 | Regression_pt: 2.1750 | Regression_sin_phi: 0.0721 | LR: 5.22e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5495s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0704 | max_tensorboard_logging: 0.1606 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2726.7 | uss_MB: 1576.4 | child_rss_MB: 22638.9 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:21:25,409] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 06:21:25,409] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:28,240] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 06:22:28,240] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:45,286] INFO : Step 26000/500000 rank0 | Train Loss: 8.4002 | Train Loss Unweighted: 3.9523 | Train Loss Components: Classification: 0.0356 | Classification_binary: 3.5510 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1681 | Regression_eta: 0.0058 | Regression_pt: 0.1690 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3966 | Classification_binary: 3.5510 | Regression_cos_phi: 0.1176 | Regression_energy: 1.8220 | Regression_eta: 0.0462 | Regression_pt: 2.3365 | Regression_sin_phi: 0.1304 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5497s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0030 | max_backward: 0.0427 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1547 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2727.3 | uss_MB: 1577.0 | child_rss_MB: 22637.3 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:22:45,288] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:22:45,362] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:23:31,775] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 06:23:31,775] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:34,683] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 06:24:34,683] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:37,795] INFO : Step 27000/500000 rank0 | Train Loss: 8.0476 | Train Loss Unweighted: 3.7336 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.3428 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1654 | Regression_eta: 0.0118 | Regression_pt: 0.1655 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3254 | Classification_binary: 3.3428 | Regression_cos_phi: 0.1034 | Regression_energy: 1.7933 | Regression_eta: 0.0942 | Regression_pt: 2.2884 | Regression_sin_phi: 0.1001 | LR: 5.74e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5162s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0366 | max_loss: 0.0027 | max_backward: 0.0400 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1343 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.6 | uss_MB: 1580.3 | child_rss_MB: 22637.9 | allocated_MB: 126.1 | reserved_MB: 632.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:25:38,079] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 06:25:38,079] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:29,989] INFO : Step 28000/500000 rank0 | Train Loss: 8.7899 | Train Loss Unweighted: 3.9411 | Train Loss Components: Classification: 0.0345 | Classification_binary: 3.5046 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1879 | Regression_eta: 0.0060 | Regression_pt: 0.1876 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3843 | Classification_binary: 3.5046 | Regression_cos_phi: 0.1118 | Regression_energy: 2.0371 | Regression_eta: 0.0476 | Regression_pt: 2.5938 | Regression_sin_phi: 0.1108 | LR: 6.01e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5673s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0378 | max_loss: 0.0032 | max_backward: 0.0423 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.1767 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.8 | uss_MB: 1580.5 | child_rss_MB: 22643.4 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:26:29,991] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:26:30,061] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:26:41,213] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 06:26:41,213] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:27:44,126] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 06:27:44,126] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:22,153] INFO : Step 29000/500000 rank0 | Train Loss: 8.6877 | Train Loss Unweighted: 4.1204 | Train Loss Components: Classification: 0.0293 | Classification_binary: 3.7091 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1763 | Regression_eta: 0.0098 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3267 | Classification_binary: 3.7091 | Regression_cos_phi: 0.0883 | Regression_energy: 1.9106 | Regression_eta: 0.0784 | Regression_pt: 2.4977 | Regression_sin_phi: 0.0769 | LR: 6.29e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5206s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0383 | max_loss: 0.0031 | max_backward: 0.0424 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1355 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2731.3 | uss_MB: 1581.0 | child_rss_MB: 22643.6 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:28:47,068] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 06:28:47,068] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:29:49,850] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 06:29:49,850] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:14,495] INFO : Step 30000/500000 rank0 | Train Loss: 8.1003 | Train Loss Unweighted: 3.7425 | Train Loss Components: Classification: 0.0373 | Classification_binary: 3.3508 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1683 | Regression_eta: 0.0048 | Regression_pt: 0.1696 | Regression_sin_phi: 0.0053 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4156 | Classification_binary: 3.3508 | Regression_cos_phi: 0.0709 | Regression_energy: 1.8241 | Regression_eta: 0.0381 | Regression_pt: 2.3442 | Regression_sin_phi: 0.0565 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5199s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0420 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2727.6 | uss_MB: 1577.3 | child_rss_MB: 22648.0 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:30:14,496] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:30:14,567] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:30:14,567] INFO : Running validation on rank0 +rank0 [2026-11-14 06:30:14,568] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:30:14,568] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:30:14,731] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 06:30:14,853] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:30:14,968] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:30:15,036] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:30:15,102] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:30:15,169] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:30:15,237] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:30:15,305] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:30:15,406] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:30:15,523] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:30:15,635] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:30:15,700] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:30:15,765] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:30:15,830] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:30:15,898] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:30:15,966] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:30:16,081] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:30:16,088] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:30:16,090] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:30:16,093] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:30:16,095] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:30:16,097] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:30:16,099] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:30:16,101] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:30:16,103] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:30:16,105] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:30:16,107] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:30:16,109] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:30:16,111] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:30:16,113] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:30:16,114] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:30:16,117] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:30:17,690] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.1003 | Valid Loss=9.2597 | Train Loss Unweighted=3.7425 | Valid Loss Unweighted=4.1063 | Stale=0 | ETA=885.2m +rank0 [2026-11-14 06:30:17,691] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:30:17,768] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:30:17,776] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:17,828] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:30:18,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:30:18,608] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:30:18,872] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:30:19,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:30:19,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:30:19,638] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:30:19,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:30:20,150] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:30:20,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:30:20,668] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:30:20,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:30:21,183] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:30:21,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:30:21,699] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:30:21,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:30:22,187] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:30:22,266] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:30:22,273] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:22,846] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:23,258] INFO : Plotted number of elements +rank0 [2026-11-14 06:30:24,616] INFO : Plotted jets +rank0 [2026-11-14 06:30:25,586] INFO : Plotted jet ratio +rank0 [2026-11-14 06:31:04,970] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 06:31:04,970] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:08,069] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 06:32:08,070] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:18,609] INFO : Step 31000/500000 rank0 | Train Loss: 8.9334 | Train Loss Unweighted: 4.3159 | Train Loss Components: Classification: 0.0311 | Classification_binary: 3.9006 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1793 | Regression_eta: 0.0067 | Regression_pt: 0.1813 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3469 | Classification_binary: 3.9006 | Regression_cos_phi: 0.0915 | Regression_energy: 1.9433 | Regression_eta: 0.0530 | Regression_pt: 2.5057 | Regression_sin_phi: 0.0924 | LR: 6.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5148s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0028 | max_backward: 0.0409 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1210 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2860.5 | uss_MB: 1710.5 | child_rss_MB: 22632.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:33:11,333] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 06:33:11,333] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:10,421] INFO : Step 32000/500000 rank0 | Train Loss: 8.0630 | Train Loss Unweighted: 3.7867 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.4000 | Regression_cos_phi: 0.0170 | Regression_energy: 0.1606 | Regression_eta: 0.0074 | Regression_pt: 0.1614 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3314 | Classification_binary: 3.4000 | Regression_cos_phi: 0.1875 | Regression_energy: 1.7405 | Regression_eta: 0.0592 | Regression_pt: 2.2311 | Regression_sin_phi: 0.1134 | LR: 7.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5436s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0027 | max_backward: 0.0444 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1529 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2860.6 | uss_MB: 1710.5 | child_rss_MB: 22625.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:34:10,423] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:34:10,491] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:34:14,258] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 06:34:14,258] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:17,412] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 06:35:17,412] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:02,839] INFO : Step 33000/500000 rank0 | Train Loss: 7.7284 | Train Loss Unweighted: 3.7381 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.3782 | Regression_cos_phi: 0.0046 | Regression_energy: 0.1524 | Regression_eta: 0.0101 | Regression_pt: 0.1578 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2922 | Classification_binary: 3.3782 | Regression_cos_phi: 0.0512 | Regression_energy: 1.6524 | Regression_eta: 0.0806 | Regression_pt: 2.1812 | Regression_sin_phi: 0.0926 | LR: 7.51e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5249s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0382 | max_loss: 0.0031 | max_backward: 0.0409 | max_optimizer_step: 0.0697 | max_tensorboard_logging: 0.1323 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.2 | uss_MB: 1604.2 | child_rss_MB: 22629.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:36:20,409] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 06:36:20,410] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:22,896] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 06:37:22,896] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:54,435] INFO : Step 34000/500000 rank0 | Train Loss: 9.4831 | Train Loss Unweighted: 4.2166 | Train Loss Components: Classification: 0.0320 | Classification_binary: 3.7427 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2062 | Regression_eta: 0.0112 | Regression_pt: 0.2094 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3567 | Classification_binary: 3.7427 | Regression_cos_phi: 0.0893 | Regression_energy: 2.2347 | Regression_eta: 0.0894 | Regression_pt: 2.8948 | Regression_sin_phi: 0.0754 | LR: 7.83e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5203s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0387 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1294 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.9 | uss_MB: 1604.8 | child_rss_MB: 22628.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:37:54,437] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:37:54,503] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:38:25,716] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 06:38:25,716] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:39:27,887] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 06:39:27,887] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:39:45,937] INFO : Step 35000/500000 rank0 | Train Loss: 8.7704 | Train Loss Unweighted: 3.9651 | Train Loss Components: Classification: 0.0465 | Classification_binary: 3.5310 | Regression_cos_phi: 0.0079 | Regression_energy: 0.1791 | Regression_eta: 0.0094 | Regression_pt: 0.1831 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.5175 | Classification_binary: 3.5310 | Regression_cos_phi: 0.0871 | Regression_energy: 1.9415 | Regression_eta: 0.0747 | Regression_pt: 2.5315 | Regression_sin_phi: 0.0871 | LR: 8.16e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5338s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0396 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1384 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2757.9 | uss_MB: 1607.9 | child_rss_MB: 22631.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 20.3 +rank0 [2026-11-14 06:40:30,968] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 06:40:30,968] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:41:33,394] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 06:41:33,394] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:41:37,205] INFO : Step 36000/500000 rank0 | Train Loss: 7.4502 | Train Loss Unweighted: 3.5235 | Train Loss Components: Classification: 0.0345 | Classification_binary: 3.1665 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1437 | Regression_eta: 0.0135 | Regression_pt: 0.1484 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3844 | Classification_binary: 3.1665 | Regression_cos_phi: 0.0930 | Regression_energy: 1.5577 | Regression_eta: 0.1076 | Regression_pt: 2.0509 | Regression_sin_phi: 0.0900 | LR: 8.50e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.4839s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0028 | max_backward: 0.0400 | max_optimizer_step: 0.0615 | max_tensorboard_logging: 0.1415 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2758.0 | uss_MB: 1608.0 | child_rss_MB: 22633.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:41:37,207] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:41:37,273] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:42:35,961] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 06:42:35,961] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:43:29,432] INFO : Step 37000/500000 rank0 | Train Loss: 8.5854 | Train Loss Unweighted: 3.8032 | Train Loss Components: Classification: 0.0361 | Classification_binary: 3.3700 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1897 | Regression_eta: 0.0120 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4020 | Classification_binary: 3.3700 | Regression_cos_phi: 0.0633 | Regression_energy: 2.0561 | Regression_eta: 0.0958 | Regression_pt: 2.5174 | Regression_sin_phi: 0.0808 | LR: 8.85e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5171s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0029 | max_backward: 0.0401 | max_optimizer_step: 0.0616 | max_tensorboard_logging: 0.1290 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2758.1 | uss_MB: 1608.1 | child_rss_MB: 22636.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 30.3 +rank0 [2026-11-14 06:43:39,268] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 06:43:39,268] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:44:41,236] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 06:44:41,236] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:45:20,268] INFO : Step 38000/500000 rank0 | Train Loss: 7.6476 | Train Loss Unweighted: 3.7467 | Train Loss Components: Classification: 0.0385 | Classification_binary: 3.3931 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1462 | Regression_eta: 0.0081 | Regression_pt: 0.1449 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.4283 | Classification_binary: 3.3931 | Regression_cos_phi: 0.0916 | Regression_energy: 1.5848 | Regression_eta: 0.0646 | Regression_pt: 2.0025 | Regression_sin_phi: 0.0828 | LR: 9.21e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5593s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0030 | max_backward: 0.0401 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1689 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.9 | uss_MB: 1604.9 | child_rss_MB: 22640.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:45:20,270] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:45:20,338] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:45:44,430] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 06:45:44,430] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:46:46,963] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 06:46:46,963] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:12,061] INFO : Step 39000/500000 rank0 | Train Loss: 7.3042 | Train Loss Unweighted: 3.6422 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.3119 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1365 | Regression_eta: 0.0071 | Regression_pt: 0.1416 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3138 | Classification_binary: 3.3119 | Regression_cos_phi: 0.1022 | Regression_energy: 1.4797 | Regression_eta: 0.0564 | Regression_pt: 1.9571 | Regression_sin_phi: 0.0831 | LR: 9.57e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5734s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0029 | max_backward: 0.0408 | max_optimizer_step: 0.0683 | max_tensorboard_logging: 0.1884 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2758.1 | uss_MB: 1608.1 | child_rss_MB: 22642.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:47:49,745] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 06:47:49,746] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:48:52,364] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 06:48:52,364] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:03,780] INFO : Step 40000/500000 rank0 | Train Loss: 8.4305 | Train Loss Unweighted: 3.9832 | Train Loss Components: Classification: 0.0350 | Classification_binary: 3.5788 | Regression_cos_phi: 0.0142 | Regression_energy: 0.1621 | Regression_eta: 0.0129 | Regression_pt: 0.1658 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3895 | Classification_binary: 3.5788 | Regression_cos_phi: 0.1560 | Regression_energy: 1.7571 | Regression_eta: 0.1026 | Regression_pt: 2.2916 | Regression_sin_phi: 0.1548 | LR: 9.94e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5415s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0034 | max_backward: 0.0409 | max_optimizer_step: 0.0798 | max_tensorboard_logging: 0.1353 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2758.4 | uss_MB: 1608.4 | child_rss_MB: 22638.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:49:03,782] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 06:49:03,854] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:49:03,856] INFO : Running validation on rank0 +rank0 [2026-11-14 06:49:03,856] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:49:03,856] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:49:04,000] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 06:49:04,121] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:49:04,237] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:49:04,305] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:49:04,371] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:49:04,436] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:49:04,502] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:49:04,568] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:49:04,671] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:49:04,791] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:49:04,908] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:49:04,976] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:49:05,042] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:49:05,108] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:49:05,177] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:49:05,245] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:49:05,346] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:49:05,353] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:49:05,355] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:49:05,357] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:49:05,359] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:49:05,361] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:49:05,363] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:49:05,365] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:49:05,367] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:49:05,370] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:49:05,371] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:49:05,373] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:49:05,376] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:49:05,378] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:49:05,380] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:49:05,382] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:49:06,939] INFO : VALIDATION | Step=40000/500000 | Train Loss=8.4305 | Valid Loss=8.8503 | Train Loss Unweighted=3.9832 | Valid Loss Unweighted=3.9182 | Stale=0 | ETA=866.2m +rank0 [2026-11-14 06:49:06,940] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:49:07,019] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:49:07,038] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:07,087] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:49:07,599] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:49:07,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:49:08,117] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:49:08,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:49:08,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:49:08,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:49:09,132] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:49:09,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:49:09,626] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:49:09,882] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:49:10,143] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:49:10,399] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:49:10,649] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:49:10,901] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:49:11,151] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:49:11,378] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:49:11,464] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:49:11,473] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:12,053] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:12,447] INFO : Plotted number of elements +rank0 [2026-11-14 06:49:13,792] INFO : Plotted jets +rank0 [2026-11-14 06:49:14,779] INFO : Plotted jet ratio +rank0 [2026-11-14 06:50:06,907] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 06:50:06,907] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:51:06,959] INFO : Step 41000/500000 rank0 | Train Loss: 7.6717 | Train Loss Unweighted: 3.6997 | Train Loss Components: Classification: 0.0256 | Classification_binary: 3.3427 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1551 | Regression_eta: 0.0041 | Regression_pt: 0.1553 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2851 | Classification_binary: 3.3427 | Regression_cos_phi: 0.0632 | Regression_energy: 1.6817 | Regression_eta: 0.0327 | Regression_pt: 2.1473 | Regression_sin_phi: 0.1190 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5687s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1849 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2778.5 | uss_MB: 1628.5 | child_rss_MB: 22644.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:51:09,835] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 06:51:09,835] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:52:12,397] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 06:52:12,397] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:52:58,548] INFO : Step 42000/500000 rank0 | Train Loss: 7.3540 | Train Loss Unweighted: 3.6255 | Train Loss Components: Classification: 0.0309 | Classification_binary: 3.2874 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1436 | Regression_eta: 0.0077 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3439 | Classification_binary: 3.2874 | Regression_cos_phi: 0.1046 | Regression_energy: 1.5570 | Regression_eta: 0.0617 | Regression_pt: 1.9168 | Regression_sin_phi: 0.0826 | LR: 1.07e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5639s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0390 | max_loss: 0.0027 | max_backward: 0.0401 | max_optimizer_step: 0.0627 | max_tensorboard_logging: 0.1836 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2779.1 | uss_MB: 1629.1 | child_rss_MB: 22643.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:52:58,550] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:52:58,621] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:53:15,336] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 06:53:15,336] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:54:18,129] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 06:54:18,130] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:54:50,715] INFO : Step 43000/500000 rank0 | Train Loss: 7.9896 | Train Loss Unweighted: 3.8364 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.4627 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1570 | Regression_eta: 0.0080 | Regression_pt: 0.1642 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3181 | Classification_binary: 3.4627 | Regression_cos_phi: 0.1010 | Regression_energy: 1.7018 | Regression_eta: 0.0634 | Regression_pt: 2.2702 | Regression_sin_phi: 0.0724 | LR: 1.11e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5135s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1246 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2781.3 | uss_MB: 1631.3 | child_rss_MB: 22646.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:55:21,339] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 06:55:21,340] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:56:24,344] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 06:56:24,344] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:56:43,175] INFO : Step 44000/500000 rank0 | Train Loss: 8.1057 | Train Loss Unweighted: 3.6378 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.2347 | Regression_cos_phi: 0.0119 | Regression_energy: 0.1725 | Regression_eta: 0.0084 | Regression_pt: 0.1732 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3141 | Classification_binary: 3.2347 | Regression_cos_phi: 0.1314 | Regression_energy: 1.8698 | Regression_eta: 0.0668 | Regression_pt: 2.3937 | Regression_sin_phi: 0.0951 | LR: 1.15e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5865s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0031 | max_backward: 0.0414 | max_optimizer_step: 0.0687 | max_tensorboard_logging: 0.1829 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.7 | uss_MB: 1631.7 | child_rss_MB: 22642.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:56:43,177] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:56:43,249] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:57:27,816] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 06:57:27,816] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:58:31,270] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 06:58:31,271] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:58:36,160] INFO : Step 45000/500000 rank0 | Train Loss: 7.0141 | Train Loss Unweighted: 3.4341 | Train Loss Components: Classification: 0.0258 | Classification_binary: 3.1068 | Regression_cos_phi: 0.0102 | Regression_energy: 0.1350 | Regression_eta: 0.0147 | Regression_pt: 0.1318 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2879 | Classification_binary: 3.1068 | Regression_cos_phi: 0.1120 | Regression_energy: 1.4632 | Regression_eta: 0.1170 | Regression_pt: 1.8226 | Regression_sin_phi: 0.1045 | LR: 1.19e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5375s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0380 | max_loss: 0.0029 | max_backward: 0.0435 | max_optimizer_step: 0.0687 | max_tensorboard_logging: 0.1436 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.7 | uss_MB: 1631.7 | child_rss_MB: 22646.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:59:34,512] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 06:59:34,512] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:00:28,129] INFO : Step 46000/500000 rank0 | Train Loss: 6.7171 | Train Loss Unweighted: 3.2995 | Train Loss Components: Classification: 0.0307 | Classification_binary: 2.9890 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1240 | Regression_eta: 0.0077 | Regression_pt: 0.1258 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3417 | Classification_binary: 2.9890 | Regression_cos_phi: 0.1064 | Regression_energy: 1.3443 | Regression_eta: 0.0610 | Regression_pt: 1.7397 | Regression_sin_phi: 0.1349 | LR: 1.23e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5308s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0025 | max_backward: 0.0390 | max_optimizer_step: 0.0628 | max_tensorboard_logging: 0.1557 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2779.3 | uss_MB: 1629.3 | child_rss_MB: 22639.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:00:28,131] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:00:28,201] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 07:00:37,543] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 07:00:37,543] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:01:40,152] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 07:01:40,152] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:02:19,981] INFO : Step 47000/500000 rank0 | Train Loss: 7.8845 | Train Loss Unweighted: 3.8084 | Train Loss Components: Classification: 0.0280 | Classification_binary: 3.4423 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1608 | Regression_eta: 0.0048 | Regression_pt: 0.1605 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3123 | Classification_binary: 3.4423 | Regression_cos_phi: 0.0562 | Regression_energy: 1.7427 | Regression_eta: 0.0386 | Regression_pt: 2.2193 | Regression_sin_phi: 0.0730 | LR: 1.27e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5422s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0029 | max_backward: 0.0411 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1473 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.4 | uss_MB: 1631.4 | child_rss_MB: 22650.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:02:43,151] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 07:02:43,152] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:03:46,294] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 07:03:46,294] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:12,418] INFO : Step 48000/500000 rank0 | Train Loss: 7.0900 | Train Loss Unweighted: 3.5414 | Train Loss Components: Classification: 0.0273 | Classification_binary: 3.2203 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1316 | Regression_eta: 0.0092 | Regression_pt: 0.1361 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3039 | Classification_binary: 3.2203 | Regression_cos_phi: 0.0923 | Regression_energy: 1.4270 | Regression_eta: 0.0733 | Regression_pt: 1.8820 | Regression_sin_phi: 0.0913 | LR: 1.31e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.5385s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0374 | max_loss: 0.0030 | max_backward: 0.0410 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1340 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.4 | uss_MB: 1631.4 | child_rss_MB: 22645.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:04:12,420] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:04:12,486] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 07:04:49,751] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 07:04:49,752] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:05:52,443] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 07:05:52,444] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:04,387] INFO : Step 49000/500000 rank0 | Train Loss: 7.3908 | Train Loss Unweighted: 3.7114 | Train Loss Components: Classification: 0.0290 | Classification_binary: 3.3788 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1379 | Regression_eta: 0.0068 | Regression_pt: 0.1399 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3225 | Classification_binary: 3.3788 | Regression_cos_phi: 0.1090 | Regression_energy: 1.4948 | Regression_eta: 0.0544 | Regression_pt: 1.9334 | Regression_sin_phi: 0.0978 | LR: 1.36e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4789s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0392 | max_optimizer_step: 0.0605 | max_tensorboard_logging: 0.1403 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.5 | uss_MB: 1631.5 | child_rss_MB: 22646.2 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:06:54,667] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 07:06:54,667] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:07:57,320] INFO : Step 50000/500000 rank0 | Train Loss: 7.7648 | Train Loss Unweighted: 3.7156 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.3485 | Regression_cos_phi: 0.0129 | Regression_energy: 0.1503 | Regression_eta: 0.0120 | Regression_pt: 0.1552 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2771 | Classification_binary: 3.3485 | Regression_cos_phi: 0.1426 | Regression_energy: 1.6292 | Regression_eta: 0.0959 | Regression_pt: 2.1451 | Regression_sin_phi: 0.1263 | LR: 1.40e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5204s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0031 | max_backward: 0.0406 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2778.7 | uss_MB: 1628.7 | child_rss_MB: 22653.9 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:07:57,322] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:07:57,392] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 07:07:57,393] INFO : Running validation on rank0 +rank0 [2026-11-14 07:07:57,393] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:07:57,393] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:07:57,540] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 07:07:57,662] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:07:57,779] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:07:57,848] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:07:57,913] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:07:57,977] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:07:58,042] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:07:58,109] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:07:58,210] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:07:58,326] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:07:58,439] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:07:58,507] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:07:58,572] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:07:58,638] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:07:58,705] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:07:58,773] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:07:58,872] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:07:58,879] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:07:58,882] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:07:58,884] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:07:58,886] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:07:58,888] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:07:58,890] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:07:58,892] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:07:58,894] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:07:58,896] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:07:58,898] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:07:58,900] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:07:58,902] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:07:58,904] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:07:58,906] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:07:58,908] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:08:00,419] INFO : VALIDATION | Step=50000/500000 | Train Loss=7.7648 | Valid Loss=8.8123 | Train Loss Unweighted=3.7156 | Valid Loss Unweighted=3.8453 | Stale=0 | ETA=848.0m +rank0 [2026-11-14 07:08:00,419] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:08:00,772] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:08:00,779] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:00,832] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:08:01,323] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:08:01,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:08:01,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:08:02,081] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:08:02,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:08:02,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:08:02,838] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:08:03,089] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:08:03,353] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:08:03,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:08:03,868] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:08:04,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:08:04,369] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:08:04,616] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:08:04,863] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:08:05,083] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:08:05,154] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:08:05,161] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:05,736] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:06,116] INFO : Plotted number of elements +rank0 [2026-11-14 07:08:07,416] INFO : Plotted jets +rank0 [2026-11-14 07:08:08,381] INFO : Plotted jet ratio +rank0 [2026-11-14 07:08:10,827] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 07:08:10,827] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:09:12,481] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 07:09:12,481] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:09:59,802] INFO : Step 51000/500000 rank0 | Train Loss: 7.2707 | Train Loss Unweighted: 3.6398 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.3106 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1393 | Regression_eta: 0.0097 | Regression_pt: 0.1382 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2827 | Classification_binary: 3.3106 | Regression_cos_phi: 0.0746 | Regression_energy: 1.5105 | Regression_eta: 0.0776 | Regression_pt: 1.9106 | Regression_sin_phi: 0.1040 | LR: 1.44e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5191s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0028 | max_backward: 0.0404 | max_optimizer_step: 0.0660 | max_tensorboard_logging: 0.1328 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2904.0 | uss_MB: 1754.1 | child_rss_MB: 22647.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:10:15,750] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 07:10:15,750] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:11:18,819] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 07:11:18,819] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:11:52,381] INFO : Step 52000/500000 rank0 | Train Loss: 7.1306 | Train Loss Unweighted: 3.5259 | Train Loss Components: Classification: 0.0269 | Classification_binary: 3.2004 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1330 | Regression_eta: 0.0089 | Regression_pt: 0.1404 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2998 | Classification_binary: 3.2004 | Regression_cos_phi: 0.0878 | Regression_energy: 1.4419 | Regression_eta: 0.0707 | Regression_pt: 1.9411 | Regression_sin_phi: 0.0887 | LR: 1.49e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5264s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0029 | max_backward: 0.0412 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.1251 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2904.0 | uss_MB: 1754.1 | child_rss_MB: 22647.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:11:52,383] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:11:52,450] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:12:22,193] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 07:12:22,193] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:13:25,158] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 07:13:25,159] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:13:44,769] INFO : Step 53000/500000 rank0 | Train Loss: 6.1628 | Train Loss Unweighted: 3.0686 | Train Loss Components: Classification: 0.0298 | Classification_binary: 2.7870 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1152 | Regression_eta: 0.0094 | Regression_pt: 0.1144 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3316 | Classification_binary: 2.7870 | Regression_cos_phi: 0.0701 | Regression_energy: 1.2489 | Regression_eta: 0.0747 | Regression_pt: 1.5811 | Regression_sin_phi: 0.0695 | LR: 1.53e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5572s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0416 | max_loss: 0.0030 | max_backward: 0.0411 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1523 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2904.1 | uss_MB: 1754.1 | child_rss_MB: 22647.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:14:28,468] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 07:14:28,468] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:15:31,423] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 07:15:31,423] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:15:37,249] INFO : Step 54000/500000 rank0 | Train Loss: 6.3708 | Train Loss Unweighted: 3.3509 | Train Loss Components: Classification: 0.0304 | Classification_binary: 3.0768 | Regression_cos_phi: 0.0043 | Regression_energy: 0.1117 | Regression_eta: 0.0081 | Regression_pt: 0.1130 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3391 | Classification_binary: 3.0768 | Regression_cos_phi: 0.0477 | Regression_energy: 1.2103 | Regression_eta: 0.0642 | Regression_pt: 1.5614 | Regression_sin_phi: 0.0713 | LR: 1.58e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5130s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0031 | max_backward: 0.0398 | max_optimizer_step: 0.0657 | max_tensorboard_logging: 0.1202 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2801.0 | uss_MB: 1651.1 | child_rss_MB: 22650.7 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:15:37,251] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:15:37,331] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:16:34,541] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 07:16:34,541] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:17:29,483] INFO : Step 55000/500000 rank0 | Train Loss: 6.9628 | Train Loss Unweighted: 3.4042 | Train Loss Components: Classification: 0.0295 | Classification_binary: 3.0818 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1280 | Regression_eta: 0.0086 | Regression_pt: 0.1347 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3285 | Classification_binary: 3.0818 | Regression_cos_phi: 0.0985 | Regression_energy: 1.3877 | Regression_eta: 0.0686 | Regression_pt: 1.8624 | Regression_sin_phi: 0.1353 | LR: 1.62e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5262s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0030 | max_backward: 0.0405 | max_optimizer_step: 0.0692 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.3 | uss_MB: 1653.4 | child_rss_MB: 22649.8 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:17:37,945] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 07:17:37,945] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:18:40,779] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 07:18:40,779] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:19:21,700] INFO : Step 56000/500000 rank0 | Train Loss: 6.6129 | Train Loss Unweighted: 3.3263 | Train Loss Components: Classification: 0.0332 | Classification_binary: 3.0291 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1231 | Regression_eta: 0.0061 | Regression_pt: 0.1240 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3703 | Classification_binary: 3.0291 | Regression_cos_phi: 0.0597 | Regression_energy: 1.3341 | Regression_eta: 0.0484 | Regression_pt: 1.7137 | Regression_sin_phi: 0.0576 | LR: 1.67e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.5187s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0404 | max_loss: 0.0032 | max_backward: 0.0408 | max_optimizer_step: 0.0689 | max_tensorboard_logging: 0.1199 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.8 | uss_MB: 1653.9 | child_rss_MB: 22652.0 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:19:21,702] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:19:21,770] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:19:44,113] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 07:19:44,113] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:20:46,996] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 07:20:46,997] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:21:14,138] INFO : Step 57000/500000 rank0 | Train Loss: 7.2629 | Train Loss Unweighted: 3.3805 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.0306 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1467 | Regression_eta: 0.0053 | Regression_pt: 0.1492 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3354 | Classification_binary: 3.0306 | Regression_cos_phi: 0.1025 | Regression_energy: 1.5903 | Regression_eta: 0.0424 | Regression_pt: 2.0623 | Regression_sin_phi: 0.0994 | LR: 1.72e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5337s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0380 | max_loss: 0.0032 | max_backward: 0.0406 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.1310 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.9 | uss_MB: 1654.0 | child_rss_MB: 22650.5 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:21:50,659] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 07:21:50,659] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:22:54,844] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 07:22:54,844] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:23:08,191] INFO : Step 58000/500000 rank0 | Train Loss: 6.2871 | Train Loss Unweighted: 3.3968 | Train Loss Components: Classification: 0.0291 | Classification_binary: 3.1311 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1035 | Regression_eta: 0.0113 | Regression_pt: 0.1003 | Regression_sin_phi: 0.0119 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3243 | Classification_binary: 3.1311 | Regression_cos_phi: 0.1069 | Regression_energy: 1.1220 | Regression_eta: 0.0898 | Regression_pt: 1.3859 | Regression_sin_phi: 0.1271 | LR: 1.76e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5748s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0031 | max_backward: 0.0405 | max_optimizer_step: 0.0643 | max_tensorboard_logging: 0.1685 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2800.5 | uss_MB: 1650.5 | child_rss_MB: 22652.5 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:23:08,193] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:23:08,265] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:23:58,236] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 07:23:58,236] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:25:00,239] INFO : Step 59000/500000 rank0 | Train Loss: 6.9724 | Train Loss Unweighted: 3.4296 | Train Loss Components: Classification: 0.0305 | Classification_binary: 3.1071 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1303 | Regression_eta: 0.0126 | Regression_pt: 0.1329 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3398 | Classification_binary: 3.1071 | Regression_cos_phi: 0.0779 | Regression_energy: 1.4128 | Regression_eta: 0.1006 | Regression_pt: 1.8367 | Regression_sin_phi: 0.0973 | LR: 1.81e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5538s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0029 | max_backward: 0.0402 | max_optimizer_step: 0.0644 | max_tensorboard_logging: 0.1523 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.3 | uss_MB: 1653.4 | child_rss_MB: 22645.8 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:25:01,339] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 07:25:01,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:26:04,263] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 07:26:04,263] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:26:52,902] INFO : Step 60000/500000 rank0 | Train Loss: 6.3340 | Train Loss Unweighted: 3.2435 | Train Loss Components: Classification: 0.0278 | Classification_binary: 2.9627 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1129 | Regression_eta: 0.0096 | Regression_pt: 0.1160 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3098 | Classification_binary: 2.9627 | Regression_cos_phi: 0.0745 | Regression_energy: 1.2235 | Regression_eta: 0.0764 | Regression_pt: 1.6038 | Regression_sin_phi: 0.0832 | LR: 1.86e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5424s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0380 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1500 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.4 | uss_MB: 1653.4 | child_rss_MB: 22621.9 | allocated_MB: 125.7 | reserved_MB: 654.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:26:52,904] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:26:52,972] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:26:52,973] INFO : Running validation on rank0 +rank0 [2026-11-14 07:26:52,973] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:26:52,973] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:26:53,125] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 07:26:53,246] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:26:53,364] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:26:53,433] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:26:53,500] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:26:53,569] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:26:53,638] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:26:53,707] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:26:53,808] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:26:53,925] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:26:54,041] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:26:54,111] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:26:54,178] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:26:54,245] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:26:54,312] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:26:54,381] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:26:54,482] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:26:54,489] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:26:54,491] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:26:54,494] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:26:54,496] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:26:54,498] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:26:54,500] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:26:54,502] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:26:54,504] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:26:54,506] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:26:54,508] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:26:54,510] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:26:54,512] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:26:54,514] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:26:54,516] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:26:54,519] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:26:56,019] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.3340 | Valid Loss=8.9484 | Train Loss Unweighted=3.2435 | Valid Loss Unweighted=3.8484 | Stale=1 | ETA=829.7m +rank0 [2026-11-14 07:26:56,019] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:26:56,420] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:26:56,429] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:26:56,483] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:26:57,012] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:26:57,270] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:26:57,528] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:26:57,781] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:26:58,038] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:26:58,289] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:26:58,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:26:58,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:26:59,075] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:26:59,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:26:59,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:26:59,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:27:00,096] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:27:00,347] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:27:00,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:27:00,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:27:00,929] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:27:00,937] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:01,522] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:01,913] INFO : Plotted number of elements +rank0 [2026-11-14 07:27:03,253] INFO : Plotted jets +rank0 [2026-11-14 07:27:04,236] INFO : Plotted jet ratio +rank0 [2026-11-14 07:27:19,667] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 07:27:19,667] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:28:22,371] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 07:28:22,371] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:28:56,780] INFO : Step 61000/500000 rank0 | Train Loss: 6.8925 | Train Loss Unweighted: 3.5547 | Train Loss Components: Classification: 0.0288 | Classification_binary: 3.2510 | Regression_cos_phi: 0.0133 | Regression_energy: 0.1184 | Regression_eta: 0.0102 | Regression_pt: 0.1236 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3209 | Classification_binary: 3.2510 | Regression_cos_phi: 0.1468 | Regression_energy: 1.2838 | Regression_eta: 0.0812 | Regression_pt: 1.7090 | Regression_sin_phi: 0.0998 | LR: 1.91e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5473s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0364 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0641 | max_tensorboard_logging: 0.1571 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2937.1 | uss_MB: 1787.5 | child_rss_MB: 22624.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:29:25,505] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 07:29:25,505] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:30:27,395] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 07:30:27,395] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:30:46,813] INFO : Step 62000/500000 rank0 | Train Loss: 6.2545 | Train Loss Unweighted: 3.3481 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.0831 | Regression_cos_phi: 0.0130 | Regression_energy: 0.1069 | Regression_eta: 0.0078 | Regression_pt: 0.1046 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2932 | Classification_binary: 3.0831 | Regression_cos_phi: 0.1427 | Regression_energy: 1.1593 | Regression_eta: 0.0623 | Regression_pt: 1.4457 | Regression_sin_phi: 0.0682 | LR: 1.95e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5465s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0364 | max_loss: 0.0027 | max_backward: 0.0415 | max_optimizer_step: 0.0623 | max_tensorboard_logging: 0.1967 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2828.1 | uss_MB: 1678.5 | child_rss_MB: 22607.9 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:30:46,815] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:30:46,880] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:31:30,741] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 07:31:30,742] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:32:33,201] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 07:32:33,201] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:32:39,639] INFO : Step 63000/500000 rank0 | Train Loss: 5.4502 | Train Loss Unweighted: 3.0795 | Train Loss Components: Classification: 0.0278 | Classification_binary: 2.8631 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0848 | Regression_eta: 0.0074 | Regression_pt: 0.0848 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3096 | Classification_binary: 2.8631 | Regression_cos_phi: 0.0759 | Regression_energy: 0.9192 | Regression_eta: 0.0592 | Regression_pt: 1.1721 | Regression_sin_phi: 0.0510 | LR: 2.00e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5259s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0028 | max_backward: 0.0412 | max_optimizer_step: 0.0654 | max_tensorboard_logging: 0.1568 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.4 | uss_MB: 1680.8 | child_rss_MB: 22602.2 | allocated_MB: 126.1 | reserved_MB: 658.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:33:35,270] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 07:33:35,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:34:31,007] INFO : Step 64000/500000 rank0 | Train Loss: 6.1490 | Train Loss Unweighted: 3.2676 | Train Loss Components: Classification: 0.0280 | Classification_binary: 3.0073 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1080 | Regression_eta: 0.0048 | Regression_pt: 0.1088 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3117 | Classification_binary: 3.0073 | Regression_cos_phi: 0.0646 | Regression_energy: 1.1712 | Regression_eta: 0.0384 | Regression_pt: 1.5036 | Regression_sin_phi: 0.0523 | LR: 2.05e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5124s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0026 | max_backward: 0.0424 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1352 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2830.2 | uss_MB: 1680.6 | child_rss_MB: 22609.9 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:34:31,009] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:34:31,080] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:34:38,699] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 07:34:38,699] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:35:41,507] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 07:35:41,507] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:36:24,144] INFO : Step 65000/500000 rank0 | Train Loss: 6.0762 | Train Loss Unweighted: 3.2428 | Train Loss Components: Classification: 0.0219 | Classification_binary: 2.9829 | Regression_cos_phi: 0.0134 | Regression_energy: 0.0996 | Regression_eta: 0.0120 | Regression_pt: 0.1015 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2441 | Classification_binary: 2.9829 | Regression_cos_phi: 0.1474 | Regression_energy: 1.0799 | Regression_eta: 0.0952 | Regression_pt: 1.4036 | Regression_sin_phi: 0.1232 | LR: 2.10e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5231s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0396 | max_loss: 0.0030 | max_backward: 0.0428 | max_optimizer_step: 0.0644 | max_tensorboard_logging: 0.1338 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2831.4 | uss_MB: 1681.7 | child_rss_MB: 22587.1 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:36:45,845] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 07:36:45,845] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:37:48,549] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 07:37:48,550] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:38:16,653] INFO : Step 66000/500000 rank0 | Train Loss: 5.9912 | Train Loss Unweighted: 3.1771 | Train Loss Components: Classification: 0.0235 | Classification_binary: 2.9208 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1047 | Regression_eta: 0.0082 | Regression_pt: 0.1036 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2617 | Classification_binary: 2.9208 | Regression_cos_phi: 0.0763 | Regression_energy: 1.1348 | Regression_eta: 0.0652 | Regression_pt: 1.4326 | Regression_sin_phi: 0.0999 | LR: 2.15e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5599s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0026 | max_backward: 0.0402 | max_optimizer_step: 0.0680 | max_tensorboard_logging: 0.1712 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2827.6 | uss_MB: 1677.9 | child_rss_MB: 22592.4 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 07:38:16,655] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:38:16,728] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:38:52,220] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 07:38:52,221] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:39:55,478] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 07:39:55,478] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:40:09,738] INFO : Step 67000/500000 rank0 | Train Loss: 6.1087 | Train Loss Unweighted: 3.1986 | Train Loss Components: Classification: 0.0250 | Classification_binary: 2.9284 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1037 | Regression_eta: 0.0226 | Regression_pt: 0.1031 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2786 | Classification_binary: 2.9284 | Regression_cos_phi: 0.0960 | Regression_energy: 1.1237 | Regression_eta: 0.1801 | Regression_pt: 1.4256 | Regression_sin_phi: 0.0765 | LR: 2.20e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5484s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.1479 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.6 | uss_MB: 1681.0 | child_rss_MB: 22599.5 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:40:59,609] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 07:40:59,609] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:42:02,758] INFO : Step 68000/500000 rank0 | Train Loss: 5.4546 | Train Loss Unweighted: 3.0139 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.7915 | Regression_cos_phi: 0.0075 | Regression_energy: 0.0912 | Regression_eta: 0.0068 | Regression_pt: 0.0885 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2540 | Classification_binary: 2.7915 | Regression_cos_phi: 0.0827 | Regression_energy: 0.9889 | Regression_eta: 0.0545 | Regression_pt: 1.2236 | Regression_sin_phi: 0.0593 | LR: 2.25e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5309s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0027 | max_backward: 0.0405 | max_optimizer_step: 0.0748 | max_tensorboard_logging: 0.1329 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2830.7 | uss_MB: 1681.0 | child_rss_MB: 22601.2 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:42:02,760] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:42:02,830] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:42:03,050] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 07:42:03,051] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:43:06,370] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 07:43:06,370] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:43:55,572] INFO : Step 69000/500000 rank0 | Train Loss: 6.1957 | Train Loss Unweighted: 3.2701 | Train Loss Components: Classification: 0.0262 | Classification_binary: 3.0011 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1064 | Regression_eta: 0.0104 | Regression_pt: 0.1007 | Regression_sin_phi: 0.0146 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2918 | Classification_binary: 3.0011 | Regression_cos_phi: 0.1178 | Regression_energy: 1.1539 | Regression_eta: 0.0828 | Regression_pt: 1.3920 | Regression_sin_phi: 0.1564 | LR: 2.30e-04 | DataLoad Time: 0.0046s | Model Forward Time: 0.5017s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0364 | max_loss: 0.0027 | max_backward: 0.0406 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1335 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2830.8 | uss_MB: 1681.1 | child_rss_MB: 22606.9 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:44:09,727] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 07:44:09,727] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:45:12,720] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 07:45:12,720] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:45:48,563] INFO : Step 70000/500000 rank0 | Train Loss: 5.7326 | Train Loss Unweighted: 3.0332 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.7872 | Regression_cos_phi: 0.0100 | Regression_energy: 0.0980 | Regression_eta: 0.0078 | Regression_pt: 0.0984 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2549 | Classification_binary: 2.7872 | Regression_cos_phi: 0.1099 | Regression_energy: 1.0622 | Regression_eta: 0.0624 | Regression_pt: 1.3599 | Regression_sin_phi: 0.0961 | LR: 2.35e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5334s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0408 | max_optimizer_step: 0.0801 | max_tensorboard_logging: 0.1430 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2827.7 | uss_MB: 1678.0 | child_rss_MB: 22612.2 | allocated_MB: 125.7 | reserved_MB: 658.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:45:48,565] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:45:48,636] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:45:48,636] INFO : Running validation on rank0 +rank0 [2026-11-14 07:45:48,636] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:45:48,636] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:45:48,781] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 07:45:48,906] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:45:49,026] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:45:49,096] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:45:49,163] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:45:49,231] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:45:49,298] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:45:49,366] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:45:49,466] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:45:49,582] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:45:49,698] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:45:49,766] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:45:49,833] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:45:49,898] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:45:49,965] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:45:50,032] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:45:50,141] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:45:50,148] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:45:50,150] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:45:50,152] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:45:50,154] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:45:50,156] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:45:50,159] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:45:50,161] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:45:50,163] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:45:50,165] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:45:50,167] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:45:50,169] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:45:50,171] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:45:50,173] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:45:50,175] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:45:50,176] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:45:51,674] INFO : VALIDATION | Step=70000/500000 | Train Loss=5.7326 | Valid Loss=9.5372 | Train Loss Unweighted=3.0332 | Valid Loss Unweighted=4.0008 | Stale=2 | ETA=811.3m +rank0 [2026-11-14 07:45:51,674] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:45:52,035] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:45:52,044] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:45:52,097] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:45:52,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:45:52,853] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:45:53,108] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:45:53,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:45:53,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:45:53,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:45:54,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:45:54,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:45:54,630] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:45:54,904] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:45:55,157] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:45:55,413] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:45:55,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:45:55,915] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:45:56,167] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:45:56,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:45:56,452] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:45:56,470] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:45:57,072] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:45:57,463] INFO : Plotted number of elements +rank0 [2026-11-14 07:45:58,807] INFO : Plotted jets +rank0 [2026-11-14 07:45:59,791] INFO : Plotted jet ratio +rank0 [2026-11-14 07:46:28,218] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 07:46:28,218] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:47:31,065] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 07:47:31,065] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:47:52,499] INFO : Step 71000/500000 rank0 | Train Loss: 5.4188 | Train Loss Unweighted: 2.9745 | Train Loss Components: Classification: 0.0288 | Classification_binary: 2.7531 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0871 | Regression_eta: 0.0039 | Regression_pt: 0.0899 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.3202 | Classification_binary: 2.7531 | Regression_cos_phi: 0.0638 | Regression_energy: 0.9445 | Regression_eta: 0.0314 | Regression_pt: 1.2429 | Regression_sin_phi: 0.0628 | LR: 2.40e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5594s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0040 | max_backward: 0.0401 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1640 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2961.5 | uss_MB: 1812.1 | child_rss_MB: 22606.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:48:34,422] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 07:48:34,422] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:49:37,549] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 07:49:37,549] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:49:45,187] INFO : Step 72000/500000 rank0 | Train Loss: 5.0680 | Train Loss Unweighted: 2.8484 | Train Loss Components: Classification: 0.0253 | Classification_binary: 2.6421 | Regression_cos_phi: 0.0106 | Regression_energy: 0.0756 | Regression_eta: 0.0128 | Regression_pt: 0.0732 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2816 | Classification_binary: 2.6421 | Regression_cos_phi: 0.1162 | Regression_energy: 0.8200 | Regression_eta: 0.1020 | Regression_pt: 1.0118 | Regression_sin_phi: 0.0943 | LR: 2.45e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5843s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0002 | max_forward: 0.0353 | max_loss: 0.0030 | max_backward: 0.0452 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1883 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2961.5 | uss_MB: 1812.1 | child_rss_MB: 22610.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:49:45,190] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 07:49:45,259] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:50:41,452] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 07:50:41,452] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:51:38,208] INFO : Step 73000/500000 rank0 | Train Loss: 5.6168 | Train Loss Unweighted: 3.0654 | Train Loss Components: Classification: 0.0242 | Classification_binary: 2.8334 | Regression_cos_phi: 0.0074 | Regression_energy: 0.0944 | Regression_eta: 0.0065 | Regression_pt: 0.0939 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2696 | Classification_binary: 2.8334 | Regression_cos_phi: 0.0817 | Regression_energy: 1.0230 | Regression_eta: 0.0518 | Regression_pt: 1.2979 | Regression_sin_phi: 0.0594 | LR: 2.50e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5438s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0379 | max_loss: 0.0031 | max_backward: 0.0403 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1371 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2961.5 | uss_MB: 1812.1 | child_rss_MB: 22609.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:51:44,941] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 07:51:44,941] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:52:47,760] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 07:52:47,760] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:53:30,441] INFO : Step 74000/500000 rank0 | Train Loss: 5.9284 | Train Loss Unweighted: 3.0440 | Train Loss Components: Classification: 0.0265 | Classification_binary: 2.7805 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1033 | Regression_eta: 0.0103 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2951 | Classification_binary: 2.7805 | Regression_cos_phi: 0.0882 | Regression_energy: 1.1194 | Regression_eta: 0.0819 | Regression_pt: 1.4538 | Regression_sin_phi: 0.1095 | LR: 2.55e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5312s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0384 | max_loss: 0.0030 | max_backward: 0.0442 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1367 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2852.8 | uss_MB: 1703.4 | child_rss_MB: 22617.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:53:30,443] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 07:53:30,515] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:53:51,104] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 07:53:51,104] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:54:53,115] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 07:54:53,115] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:55:21,860] INFO : Step 75000/500000 rank0 | Train Loss: 4.8602 | Train Loss Unweighted: 2.8178 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.6287 | Regression_cos_phi: 0.0093 | Regression_energy: 0.0714 | Regression_eta: 0.0094 | Regression_pt: 0.0674 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2545 | Classification_binary: 2.6287 | Regression_cos_phi: 0.1025 | Regression_energy: 0.7735 | Regression_eta: 0.0747 | Regression_pt: 0.9324 | Regression_sin_phi: 0.0938 | LR: 2.60e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5828s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0381 | max_loss: 0.0031 | max_backward: 0.0427 | max_optimizer_step: 0.0748 | max_tensorboard_logging: 0.1507 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2855.5 | uss_MB: 1706.1 | child_rss_MB: 22622.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:55:57,259] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 07:55:57,260] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:56:59,897] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 07:56:59,897] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:57:14,437] INFO : Step 76000/500000 rank0 | Train Loss: 5.0480 | Train Loss Unweighted: 2.8762 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.6785 | Regression_cos_phi: 0.0074 | Regression_energy: 0.0792 | Regression_eta: 0.0049 | Regression_pt: 0.0782 | Regression_sin_phi: 0.0055 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2511 | Classification_binary: 2.6785 | Regression_cos_phi: 0.0810 | Regression_energy: 0.8586 | Regression_eta: 0.0393 | Regression_pt: 1.0804 | Regression_sin_phi: 0.0590 | LR: 2.65e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5125s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0035 | max_backward: 0.0400 | max_optimizer_step: 0.0616 | max_tensorboard_logging: 0.1364 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2855.5 | uss_MB: 1706.1 | child_rss_MB: 22627.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:57:14,439] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 07:57:14,506] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:58:03,163] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 07:58:03,163] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:59:06,518] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 07:59:06,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:59:07,634] INFO : Step 77000/500000 rank0 | Train Loss: 5.0269 | Train Loss Unweighted: 2.6675 | Train Loss Components: Classification: 0.0210 | Classification_binary: 2.4533 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0871 | Regression_eta: 0.0059 | Regression_pt: 0.0876 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2336 | Classification_binary: 2.4533 | Regression_cos_phi: 0.0740 | Regression_energy: 0.9439 | Regression_eta: 0.0470 | Regression_pt: 1.2111 | Regression_sin_phi: 0.0640 | LR: 2.70e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5391s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0031 | max_backward: 0.0413 | max_optimizer_step: 0.0763 | max_tensorboard_logging: 0.1400 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.8 | uss_MB: 1706.4 | child_rss_MB: 22623.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 08:00:10,616] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 08:00:10,616] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:01:00,932] INFO : Step 78000/500000 rank0 | Train Loss: 4.6779 | Train Loss Unweighted: 2.6955 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.5148 | Regression_cos_phi: 0.0098 | Regression_energy: 0.0699 | Regression_eta: 0.0039 | Regression_pt: 0.0700 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2285 | Classification_binary: 2.5148 | Regression_cos_phi: 0.1081 | Regression_energy: 0.7572 | Regression_eta: 0.0313 | Regression_pt: 0.9677 | Regression_sin_phi: 0.0702 | LR: 2.75e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5134s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0026 | max_backward: 0.0389 | max_optimizer_step: 0.0631 | max_tensorboard_logging: 0.1368 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.8 | uss_MB: 1706.4 | child_rss_MB: 22631.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:01:00,934] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 08:01:01,004] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 08:01:14,230] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 08:01:14,230] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:17,304] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 08:02:17,304] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:53,391] INFO : Step 79000/500000 rank0 | Train Loss: 5.0982 | Train Loss Unweighted: 2.8918 | Train Loss Components: Classification: 0.0194 | Classification_binary: 2.6902 | Regression_cos_phi: 0.0067 | Regression_energy: 0.0791 | Regression_eta: 0.0083 | Regression_pt: 0.0804 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2162 | Classification_binary: 2.6902 | Regression_cos_phi: 0.0738 | Regression_energy: 0.8569 | Regression_eta: 0.0664 | Regression_pt: 1.1110 | Regression_sin_phi: 0.0838 | LR: 2.80e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5699s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0031 | max_backward: 0.0406 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1844 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.4 | uss_MB: 1706.0 | child_rss_MB: 22621.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:03:20,370] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 08:03:20,370] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:04:23,295] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 08:04:23,295] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:04:46,068] INFO : Step 80000/500000 rank0 | Train Loss: 4.2202 | Train Loss Unweighted: 2.4521 | Train Loss Components: Classification: 0.0213 | Classification_binary: 2.2897 | Regression_cos_phi: 0.0089 | Regression_energy: 0.0618 | Regression_eta: 0.0053 | Regression_pt: 0.0596 | Regression_sin_phi: 0.0054 | Task Weights: Classification: 11.1389 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0118 | Regression_energy: 10.8398 | Regression_eta: 7.9622 | Regression_pt: 13.8240 | Regression_sin_phi: 10.7244 | Task Weighted Losses: Classification: 0.2369 | Classification_binary: 2.2897 | Regression_cos_phi: 0.0985 | Regression_energy: 0.6704 | Regression_eta: 0.0424 | Regression_pt: 0.8241 | Regression_sin_phi: 0.0583 | LR: 2.85e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5206s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1269 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.5 | uss_MB: 1706.1 | child_rss_MB: 22626.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:04:46,070] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 08:04:46,139] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 08:04:46,139] INFO : Running validation on rank0 +rank0 [2026-11-14 08:04:46,139] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 08:04:46,139] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 08:04:46,288] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 08:04:46,412] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 08:04:46,531] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 08:04:46,601] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 08:04:46,669] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 08:04:46,735] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 08:04:46,804] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 08:04:46,873] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 08:04:46,978] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 08:04:47,097] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 08:04:47,217] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 08:04:47,285] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 08:04:47,352] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 08:04:47,421] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 08:04:47,489] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 08:04:47,559] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 08:04:47,662] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 08:04:47,669] INFO : plotted pt_target histogram +rank0 [2026-11-14 08:04:47,671] INFO : plotted pt_pred histogram +rank0 [2026-11-14 08:04:47,673] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 08:04:47,675] INFO : plotted eta_target histogram +rank0 [2026-11-14 08:04:47,677] INFO : plotted eta_pred histogram +rank0 [2026-11-14 08:04:47,679] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 08:04:47,681] INFO : plotted sphi_target histogram +rank0 [2026-11-14 08:04:47,683] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 08:04:47,685] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 08:04:47,687] INFO : plotted cphi_target histogram +rank0 [2026-11-14 08:04:47,689] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 08:04:47,691] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 08:04:47,692] INFO : plotted energy_target histogram +rank0 [2026-11-14 08:04:47,694] INFO : plotted energy_pred histogram +rank0 [2026-11-14 08:04:47,697] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 08:04:49,196] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.2202 | Valid Loss=10.4248 | Train Loss Unweighted=2.4521 | Valid Loss Unweighted=4.4044 | Stale=3 | ETA=792.9m +rank0 [2026-11-14 08:04:49,197] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 08:04:49,700] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 08:04:49,707] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 08:04:49,761] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 08:04:50,278] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 08:04:50,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 08:04:50,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 08:04:51,036] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 08:04:51,292] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 08:04:51,549] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 08:04:51,809] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 08:04:52,062] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 08:04:52,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 08:04:52,561] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 08:04:52,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 08:04:53,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 08:04:53,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 08:04:53,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 08:04:53,834] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 08:04:54,063] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 08:04:54,111] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 08:04:54,118] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:04:54,702] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_15mm_seed12345_20261114_053319_103794/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:04:55,101] INFO : Plotted number of elements +rank0 [2026-11-14 08:04:56,450] INFO : Plotted jets +rank0 [2026-11-14 08:04:57,431] INFO : Plotted jet ratio +rank0 [2026-11-14 08:04:57,898] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 08:04:57,898] INFO : Training completed. Total time on device 0: 151.175min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..36e30ad56e142559c8490c8d11d620d4ddcf5d8c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:411d95a1b179ced049073c4d1c05ce23494e43d509d696326a5bbd8c7c22bee0 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-80000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-80000.pth new file mode 100644 index 0000000000000000000000000000000000000000..8f6ee2cf77d3fb9406ad91b13c49b64b57632cf1 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-80000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c4f6e2eb1c8d59a94977a4b21448113338aa7b7017264a87f9af20af61d5003 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..8e368b7e0bf644ae7031313453ad5ef4161e6165 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 3.8630948066711426, "Classification": 0.03288106247782707, "Regression_pt": 0.22533433139324188, "Regression_eta": 0.008127544075250626, "Regression_sin_phi": 0.009553736075758934, "Regression_cos_phi": 0.010791029781103134, "Regression_energy": 0.2252025455236435, "Total": 9.987894058227539, "Total_unweighted": 4.374985055997968}, "train_group": {}, "valid": {"Classification_binary": 3.8567371368408203, "Classification": 0.03816668689250946, "Regression_pt": 0.2143709808588028, "Regression_eta": 0.01088318694382906, "Regression_sin_phi": 0.010720226913690567, "Regression_cos_phi": 0.010407617315649986, "Regression_energy": 0.20975685119628906, "Total": 9.757105827331543, "Total_unweighted": 4.351042686961591, "metrics/particle/class_1/efficiency": 0.8882300853729248, "metrics/particle/class_1/pid_accuracy": 0.9812692999839783, "metrics/particle/class_2/efficiency": 0.1990898698568344, "metrics/particle/class_2/pid_accuracy": 0.3199999928474426, "metrics/particle/class_3/efficiency": 0.70489901304245, "metrics/particle/class_3/pid_accuracy": 0.9783413410186768, "metrics/particle/class_4/efficiency": 0.8233438730239868, "metrics/particle/class_4/pid_accuracy": 0.038314174860715866, "metrics/particle/class_5/efficiency": 0.9471153616905212, "metrics/particle/class_5/pid_accuracy": 0.12690354883670807, "metrics/particle/count/bias_mean": -0.7440000176429749, "metrics/particle/count/mae": 4.624000072479248, "metrics/particle/count/prediction_mean": 94.98400115966797, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.0779944583773613, "metrics/particle/event/energy_response_mean": 0.9928831458091736, "metrics/particle/event/met_abs_error": 13.850982666015625, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07713750004768372, "metrics/particle/event/scalar_pt_response_mean": 0.9972812533378601, "metrics/particle/event/vector_pt_closure": 0.09204941242933273, "metrics/particle/matched/delta_eta_abs_mean": 0.021454032510519028, "metrics/particle/matched/delta_phi_abs_mean": 0.019167881458997726, "metrics/particle/matched/delta_r_mean": 0.031999628990888596, "metrics/particle/matched/energy_relative_abs_mean": 0.09744642674922943, "metrics/particle/matched/pid_accuracy": 0.9444873332977295, "metrics/particle/matched/pt_relative_abs_mean": 0.09873255342245102, "metrics/particle/matching/duplicate_fraction": 0.04426008462905884, "metrics/particle/matching/efficiency": 0.7579809427261353, "metrics/particle/matching/f1": 0.7609379291534424, "metrics/particle/matching/purity": 0.7639181613922119, "metrics/particle/matching/target_coverage_dr0p05": 0.6323750615119934, "metrics/particle/matching/target_coverage_dr0p10": 0.7941668033599854, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6683102250099182, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8157696723937988}, "step_train_time": 0.5207734107971191, "step_valid_time": 2.855466066976078, "step_total_time": 1135.4691469669342, "elapsed_training_loop_seconds": 1135.4691469669342, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036460102645581, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19958070971796915, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1494, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1320, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1234, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.890087660148348, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8835341365461847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8867987907289218, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11646586345381527, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8320971004720162, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8259705488621151, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8290225058784011, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.890087660148348, "step_test_time": 5.234025668003596, "step_plot_time": 3.8172801270266064, "step_validation_cycle_time": 12.422154666972347} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..b1fa7563c64c03c44b71fbf4c77705641c4be915 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 4.102086067199707, "Classification": 0.0320911668241024, "Regression_pt": 0.21445871889591217, "Regression_eta": 0.010198790580034256, "Regression_sin_phi": 0.010443133302032948, "Regression_cos_phi": 0.010332291014492512, "Regression_energy": 0.2104521095752716, "Total": 9.93392562866211, "Total_unweighted": 4.590062277391553}, "train_group": {}, "valid": {"Classification_binary": 3.704224109649658, "Classification": 0.03633243963122368, "Regression_pt": 0.20795148611068726, "Regression_eta": 0.009710697457194328, "Regression_sin_phi": 0.009923305362462997, "Regression_cos_phi": 0.009677466005086899, "Regression_energy": 0.20412170886993408, "Total": 9.41049575805664, "Total_unweighted": 4.1819412130862474, "metrics/particle/class_1/efficiency": 0.9038053154945374, "metrics/particle/class_1/pid_accuracy": 0.9809066653251648, "metrics/particle/class_2/efficiency": 0.22070534527301788, "metrics/particle/class_2/pid_accuracy": 0.39175257086753845, "metrics/particle/class_3/efficiency": 0.7288627028465271, "metrics/particle/class_3/pid_accuracy": 0.9801140427589417, "metrics/particle/class_4/efficiency": 0.8675078749656677, "metrics/particle/class_4/pid_accuracy": 0.07636363804340363, "metrics/particle/class_5/efficiency": 0.9567307829856873, "metrics/particle/class_5/pid_accuracy": 0.21608039736747742, "metrics/particle/count/bias_mean": -0.6439999938011169, "metrics/particle/count/mae": 4.52400016784668, "metrics/particle/count/prediction_mean": 95.08399963378906, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.0693458691239357, "metrics/particle/event/energy_response_mean": 0.9705804586410522, "metrics/particle/event/met_abs_error": 12.181323051452637, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06858847290277481, "metrics/particle/event/scalar_pt_response_mean": 0.9702227115631104, "metrics/particle/event/vector_pt_closure": 0.08426566421985626, "metrics/particle/matched/delta_eta_abs_mean": 0.018519096076488495, "metrics/particle/matched/delta_phi_abs_mean": 0.017991188913583755, "metrics/particle/matched/delta_r_mean": 0.028714943677186966, "metrics/particle/matched/energy_relative_abs_mean": 0.09613052010536194, "metrics/particle/matched/pid_accuracy": 0.9467719197273254, "metrics/particle/matched/pt_relative_abs_mean": 0.09021255373954773, "metrics/particle/matching/duplicate_fraction": 0.04072188958525658, "metrics/particle/matching/efficiency": 0.7779542207717896, "metrics/particle/matching/f1": 0.7805798649787903, "metrics/particle/matching/purity": 0.7832232713699341, "metrics/particle/matching/target_coverage_dr0p05": 0.6799682378768921, "metrics/particle/matching/target_coverage_dr0p10": 0.8127193450927734, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6838542819023132, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8271769881248474}, "step_train_time": 0.560455322265625, "step_valid_time": 3.129619991988875, "step_total_time": 2283.2694709300995, "elapsed_training_loop_seconds": 2283.2694709300995, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9900405981739295, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18852730434944998, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1495, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1328, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1263, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8954821308159137, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8882943143812709, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8918737407656145, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1117056856187291, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.851652056641942, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8448160535117056, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8482202820685023, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8954821308159137, "step_test_time": 4.920906765968539, "step_plot_time": 3.3631228379672393, "step_validation_cycle_time": 12.003186237008777} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..63c967087aec73ac35fcd61bb01296002dcfebc3 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 3.705228328704834, "Classification": 0.039329782128334045, "Regression_pt": 0.16123275458812714, "Regression_eta": 0.009348615072667599, "Regression_sin_phi": 0.008842917159199715, "Regression_cos_phi": 0.007017430383712053, "Regression_energy": 0.15908727049827576, "Total": 8.284530639648438, "Total_unweighted": 4.09008709853515}, "train_group": {}, "valid": {"Classification_binary": 3.599651336669922, "Classification": 0.03515272215008736, "Regression_pt": 0.19594694674015045, "Regression_eta": 0.009384261444211006, "Regression_sin_phi": 0.009611602872610092, "Regression_cos_phi": 0.00940120592713356, "Regression_energy": 0.19224275648593903, "Total": 8.993560791015625, "Total_unweighted": 4.051390832290053, "metrics/particle/class_1/efficiency": 0.9082300662994385, "metrics/particle/class_1/pid_accuracy": 0.9804151058197021, "metrics/particle/class_2/efficiency": 0.2542662024497986, "metrics/particle/class_2/pid_accuracy": 0.43624159693717957, "metrics/particle/class_3/efficiency": 0.7540825009346008, "metrics/particle/class_3/pid_accuracy": 0.981291651725769, "metrics/particle/class_4/efficiency": 0.8643532991409302, "metrics/particle/class_4/pid_accuracy": 0.12773722410202026, "metrics/particle/class_5/efficiency": 0.9375, "metrics/particle/class_5/pid_accuracy": 0.2410256415605545, "metrics/particle/count/bias_mean": 1.4639999866485596, "metrics/particle/count/mae": 4.495999813079834, "metrics/particle/count/prediction_mean": 97.19200134277344, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.06296244263648987, "metrics/particle/event/energy_response_mean": 0.9819154143333435, "metrics/particle/event/met_abs_error": 9.91182804107666, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06206910312175751, "metrics/particle/event/scalar_pt_response_mean": 0.979443371295929, "metrics/particle/event/vector_pt_closure": 0.0682765319943428, "metrics/particle/matched/delta_eta_abs_mean": 0.018664821982383728, "metrics/particle/matched/delta_phi_abs_mean": 0.01772201433777809, "metrics/particle/matched/delta_r_mean": 0.028725501149892807, "metrics/particle/matched/energy_relative_abs_mean": 0.09755486994981766, "metrics/particle/matched/pid_accuracy": 0.9480587840080261, "metrics/particle/matched/pt_relative_abs_mean": 0.09208757430315018, "metrics/particle/matching/duplicate_fraction": 0.03625812754034996, "metrics/particle/matching/efficiency": 0.793205738067627, "metrics/particle/matching/f1": 0.7871863842010498, "metrics/particle/matching/purity": 0.781257688999176, "metrics/particle/matching/target_coverage_dr0p05": 0.6899548768997192, "metrics/particle/matching/target_coverage_dr0p10": 0.8248370289802551, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6920859217643738, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8423032164573669}, "step_train_time": 0.49178624153137207, "step_valid_time": 3.059775879024528, "step_total_time": 3412.903409719467, "elapsed_training_loop_seconds": 3412.903409719467, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9897043915024859, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17052851485718934, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1339, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1299, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9028995279838166, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9010767160161507, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.901987201077804, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09892328398384927, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8759271746459879, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8741588156123823, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8750421017177501, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9028995279838166, "step_test_time": 4.818784476956353, "step_plot_time": 3.4386611520312726, "step_validation_cycle_time": 11.862137051997706} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..bf8df820748a1fcd972ff6dba73b6b3922d6ad15 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 3.1408543586730957, "Classification": 0.020106548443436623, "Regression_pt": 0.16018638014793396, "Regression_eta": 0.008193870075047016, "Regression_sin_phi": 0.009697549976408482, "Regression_cos_phi": 0.010714760981500149, "Regression_energy": 0.15636996924877167, "Total": 7.503188610076904, "Total_unweighted": 3.5061234375461936}, "train_group": {}, "valid": {"Classification_binary": 3.4016692638397217, "Classification": 0.031210899353027344, "Regression_pt": 0.18648111820220947, "Regression_eta": 0.009395941160619259, "Regression_sin_phi": 0.009360295720398426, "Regression_cos_phi": 0.009092030115425587, "Regression_energy": 0.1837851107120514, "Total": 8.52645206451416, "Total_unweighted": 3.830994659103453, "metrics/particle/class_1/efficiency": 0.9100884795188904, "metrics/particle/class_1/pid_accuracy": 0.9781213402748108, "metrics/particle/class_2/efficiency": 0.2724687159061432, "metrics/particle/class_2/pid_accuracy": 0.48851773142814636, "metrics/particle/class_3/efficiency": 0.7436467409133911, "metrics/particle/class_3/pid_accuracy": 0.9794698357582092, "metrics/particle/class_4/efficiency": 0.8769716024398804, "metrics/particle/class_4/pid_accuracy": 0.21223022043704987, "metrics/particle/class_5/efficiency": 0.9375, "metrics/particle/class_5/pid_accuracy": 0.4205128252506256, "metrics/particle/count/bias_mean": 1.1119999885559082, "metrics/particle/count/mae": 4.4720001220703125, "metrics/particle/count/prediction_mean": 96.83999633789062, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.05438916012644768, "metrics/particle/event/energy_response_mean": 0.9755250811576843, "metrics/particle/event/met_abs_error": 8.839950561523438, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05323042348027229, "metrics/particle/event/scalar_pt_response_mean": 0.9806205034255981, "metrics/particle/event/vector_pt_closure": 0.059922706335783005, "metrics/particle/matched/delta_eta_abs_mean": 0.01952880248427391, "metrics/particle/matched/delta_phi_abs_mean": 0.018194591626524925, "metrics/particle/matched/delta_r_mean": 0.02972201630473137, "metrics/particle/matched/energy_relative_abs_mean": 0.09372425824403763, "metrics/particle/matched/pid_accuracy": 0.9492921829223633, "metrics/particle/matched/pt_relative_abs_mean": 0.09042926132678986, "metrics/particle/matching/duplicate_fraction": 0.0389508455991745, "metrics/particle/matching/efficiency": 0.7910746932029724, "metrics/particle/matching/f1": 0.7865065932273865, "metrics/particle/matching/purity": 0.7819908857345581, "metrics/particle/matching/target_coverage_dr0p05": 0.6693966388702393, "metrics/particle/matching/target_coverage_dr0p10": 0.821118175983429, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.704245388507843, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8467324376106262}, "step_train_time": 0.4898185729980469, "step_valid_time": 3.1070144299883395, "step_total_time": 4550.110650539398, "elapsed_training_loop_seconds": 4550.110650539398, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.989464022615093, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1496747348865115, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1343, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1326, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9055967633175994, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9074324324324324, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9065136685791427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09256756756756757, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8941335131490222, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8959459459459459, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8950388120148498, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9055967633175994, "step_test_time": 5.172157426015474, "step_plot_time": 3.3820870670024306, "step_validation_cycle_time": 12.1800378789776} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..d04521fb9ead31ae95d718bb3a7d049b51b0f420 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 3.1658577919006348, "Classification": 0.03157907724380493, "Regression_pt": 0.16918037831783295, "Regression_eta": 0.008754899725317955, "Regression_sin_phi": 0.00951982382684946, "Regression_cos_phi": 0.011095494963228703, "Regression_energy": 0.1725587695837021, "Total": 7.958062171936035, "Total_unweighted": 3.568546235561371}, "train_group": {}, "valid": {"Classification_binary": 3.340954542160034, "Classification": 0.030104028061032295, "Regression_pt": 0.18473657965660095, "Regression_eta": 0.009159192442893982, "Regression_sin_phi": 0.0088784359395504, "Regression_cos_phi": 0.00866912491619587, "Regression_energy": 0.18305759131908417, "Total": 8.410080909729004, "Total_unweighted": 3.765559494495392, "metrics/particle/class_1/efficiency": 0.9101769924163818, "metrics/particle/class_1/pid_accuracy": 0.9771512150764465, "metrics/particle/class_2/efficiency": 0.2684869170188904, "metrics/particle/class_2/pid_accuracy": 0.49788135290145874, "metrics/particle/class_3/efficiency": 0.7558218240737915, "metrics/particle/class_3/pid_accuracy": 0.980951189994812, "metrics/particle/class_4/efficiency": 0.8706624507904053, "metrics/particle/class_4/pid_accuracy": 0.239130437374115, "metrics/particle/class_5/efficiency": 0.9519230723381042, "metrics/particle/class_5/pid_accuracy": 0.5151515007019043, "metrics/particle/count/bias_mean": 0.41200000047683716, "metrics/particle/count/mae": 4.188000202178955, "metrics/particle/count/prediction_mean": 96.13999938964844, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.05068395659327507, "metrics/particle/event/energy_response_mean": 1.0002630949020386, "metrics/particle/event/met_abs_error": 9.432111740112305, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05048464611172676, "metrics/particle/event/scalar_pt_response_mean": 1.003892183303833, "metrics/particle/event/vector_pt_closure": 0.061941083520650864, "metrics/particle/matched/delta_eta_abs_mean": 0.019546303898096085, "metrics/particle/matched/delta_phi_abs_mean": 0.017990384250879288, "metrics/particle/matched/delta_r_mean": 0.029517998918890953, "metrics/particle/matched/energy_relative_abs_mean": 0.09458202868700027, "metrics/particle/matched/pid_accuracy": 0.951346218585968, "metrics/particle/matched/pt_relative_abs_mean": 0.09282438457012177, "metrics/particle/matching/duplicate_fraction": 0.038111086934804916, "metrics/particle/matching/efficiency": 0.7961307168006897, "metrics/particle/matching/f1": 0.7944211363792419, "metrics/particle/matching/purity": 0.7927189469337463, "metrics/particle/matching/target_coverage_dr0p05": 0.6721962094306946, "metrics/particle/matching/target_coverage_dr0p10": 0.8237923979759216, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7062928080558777, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8491559624671936}, "step_train_time": 0.5020225048065186, "step_valid_time": 3.1076356699923053, "step_total_time": 5697.363931655884, "elapsed_training_loop_seconds": 5697.363931655884, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0047546656453985, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14615598729668444, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1344, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1324, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9062710721510452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9068825910931174, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9065767284991568, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0931174089068826, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8927848954821308, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.893387314439946, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8930860033726813, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9062710721510452, "step_test_time": 4.909133492037654, "step_plot_time": 3.437754301005043, "step_validation_cycle_time": 12.01342380704591} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_60000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_60000.json new file mode 100644 index 0000000000000000000000000000000000000000..e709eec1b526dbd30045f5213ac7992987fadec1 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_60000.json @@ -0,0 +1 @@ +{"epoch": 106.76156583629893, "train": {"Classification_binary": 3.1247618198394775, "Classification": 0.02223672904074192, "Regression_pt": 0.10636690258979797, "Regression_eta": 0.012330431491136551, "Regression_sin_phi": 0.008082554675638676, "Regression_cos_phi": 0.00800115242600441, "Regression_energy": 0.10791835933923721, "Total": 6.247122287750244, "Total_unweighted": 3.3896979494020343}, "train_group": {}, "valid": {"Classification_binary": 3.333871603012085, "Classification": 0.030135568231344223, "Regression_pt": 0.19959351420402527, "Regression_eta": 0.00896997656673193, "Regression_sin_phi": 0.008701968006789684, "Regression_cos_phi": 0.008404741063714027, "Regression_energy": 0.19715064764022827, "Total": 8.749610900878906, "Total_unweighted": 3.7868280187249184, "metrics/particle/class_1/efficiency": 0.9128318428993225, "metrics/particle/class_1/pid_accuracy": 0.9789626598358154, "metrics/particle/class_2/efficiency": 0.2548350393772125, "metrics/particle/class_2/pid_accuracy": 0.5223214030265808, "metrics/particle/class_3/efficiency": 0.7463523149490356, "metrics/particle/class_3/pid_accuracy": 0.9810978770256042, "metrics/particle/class_4/efficiency": 0.8769716024398804, "metrics/particle/class_4/pid_accuracy": 0.2374100685119629, "metrics/particle/class_5/efficiency": 0.9567307829856873, "metrics/particle/class_5/pid_accuracy": 0.4673366844654083, "metrics/particle/count/bias_mean": -0.2759999930858612, "metrics/particle/count/mae": 4.395999908447266, "metrics/particle/count/prediction_mean": 95.4520034790039, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.05802614614367485, "metrics/particle/event/energy_response_mean": 0.9946721792221069, "metrics/particle/event/met_abs_error": 10.247415542602539, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05707884207367897, "metrics/particle/event/scalar_pt_response_mean": 0.9981414675712585, "metrics/particle/event/vector_pt_closure": 0.06701762229204178, "metrics/particle/matched/delta_eta_abs_mean": 0.018861589953303337, "metrics/particle/matched/delta_phi_abs_mean": 0.019151167944073677, "metrics/particle/matched/delta_r_mean": 0.029905131086707115, "metrics/particle/matched/energy_relative_abs_mean": 0.09372735768556595, "metrics/particle/matched/pid_accuracy": 0.9528053402900696, "metrics/particle/matched/pt_relative_abs_mean": 0.0891127809882164, "metrics/particle/matching/duplicate_fraction": 0.038511503487825394, "metrics/particle/matching/efficiency": 0.7924118041992188, "metrics/particle/matching/f1": 0.7935557961463928, "metrics/particle/matching/purity": 0.7947031259536743, "metrics/particle/matching/target_coverage_dr0p05": 0.6770433187484741, "metrics/particle/matching/target_coverage_dr0p10": 0.8240430951118469, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7069196105003357, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8461474180221558}, "step_train_time": 0.4944789409637451, "step_valid_time": 3.079733378952369, "step_total_time": 6840.5439484119415, "elapsed_training_loop_seconds": 6840.5439484119415, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9992968311790258, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15546126338936084, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1340, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1316, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9035738368172623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9035738368172623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9035738368172623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0964261631827377, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8873904248145651, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8873904248145651, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8873904248145651, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9035738368172623, "step_test_time": 4.835187537013553, "step_plot_time": 3.3355658109649085, "step_validation_cycle_time": 11.727952384040691} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_70000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_70000.json new file mode 100644 index 0000000000000000000000000000000000000000..7b0b44c966c4a940c860aa96203b29bf17563420 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_70000.json @@ -0,0 +1 @@ +{"epoch": 124.55516014234875, "train": {"Classification_binary": 2.967832326889038, "Classification": 0.024700013920664787, "Regression_pt": 0.10838543623685837, "Regression_eta": 0.012993009760975838, "Regression_sin_phi": 0.008134616538882256, "Regression_cos_phi": 0.00911618396639824, "Regression_energy": 0.11194070428609848, "Total": 6.206249237060547, "Total_unweighted": 3.243102291598916}, "train_group": {}, "valid": {"Classification_binary": 3.46125864982605, "Classification": 0.031051037833094597, "Regression_pt": 0.2159058302640915, "Regression_eta": 0.009124199859797955, "Regression_sin_phi": 0.008525495417416096, "Regression_cos_phi": 0.008264053612947464, "Regression_energy": 0.21403607726097107, "Total": 9.287193298339844, "Total_unweighted": 3.9481653440743685, "metrics/particle/class_1/efficiency": 0.9088495373725891, "metrics/particle/class_1/pid_accuracy": 0.9783836603164673, "metrics/particle/class_2/efficiency": 0.286689430475235, "metrics/particle/class_2/pid_accuracy": 0.545634925365448, "metrics/particle/class_3/efficiency": 0.7253841161727905, "metrics/particle/class_3/pid_accuracy": 0.97908616065979, "metrics/particle/class_4/efficiency": 0.8643532991409302, "metrics/particle/class_4/pid_accuracy": 0.22627736628055573, "metrics/particle/class_5/efficiency": 0.9471153616905212, "metrics/particle/class_5/pid_accuracy": 0.5076141953468323, "metrics/particle/count/bias_mean": -1.9880000352859497, "metrics/particle/count/mae": 5.0279998779296875, "metrics/particle/count/prediction_mean": 93.73999786376953, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.0594702810049057, "metrics/particle/event/energy_response_mean": 0.9804975986480713, "metrics/particle/event/met_abs_error": 8.819392204284668, "metrics/particle/event/scalar_pt_relative_abs_error": 0.057198382914066315, "metrics/particle/event/scalar_pt_response_mean": 0.9870278239250183, "metrics/particle/event/vector_pt_closure": 0.06030256673693657, "metrics/particle/matched/delta_eta_abs_mean": 0.0201585553586483, "metrics/particle/matched/delta_phi_abs_mean": 0.018282845616340637, "metrics/particle/matched/delta_r_mean": 0.0302965696901083, "metrics/particle/matched/energy_relative_abs_mean": 0.09907244890928268, "metrics/particle/matched/pid_accuracy": 0.9510985612869263, "metrics/particle/matched/pt_relative_abs_mean": 0.09006631374359131, "metrics/particle/matching/duplicate_fraction": 0.0380200557410717, "metrics/particle/matching/efficiency": 0.783553421497345, "metrics/particle/matching/f1": 0.7917748689651489, "metrics/particle/matching/purity": 0.8001706600189209, "metrics/particle/matching/target_coverage_dr0p05": 0.6525154709815979, "metrics/particle/matching/target_coverage_dr0p10": 0.812761127948761, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6951362490653992, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8349907994270325}, "step_train_time": 0.5110113620758057, "step_valid_time": 3.065572116000112, "step_total_time": 7990.41991019249, "elapsed_training_loop_seconds": 7990.41991019249, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9947837194070112, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1443514005992328, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1478, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1339, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1311, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9028995279838166, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9059539918809202, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9044241810199257, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09404600811907984, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8840188806473365, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8870094722598105, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8855116514690983, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9028995279838166, "step_test_time": 4.980065969983116, "step_plot_time": 3.360940505983308, "step_validation_cycle_time": 11.894633913994767} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_80000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_80000.json new file mode 100644 index 0000000000000000000000000000000000000000..9666194e4d0c9cd79d45dc066f1d03f9545e4925 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/history/step_80000.json @@ -0,0 +1 @@ +{"epoch": 142.34875444839858, "train": {"Classification_binary": 2.4867587089538574, "Classification": 0.02230778895318508, "Regression_pt": 0.07055599987506866, "Regression_eta": 0.009214052930474281, "Regression_sin_phi": 0.008140765130519867, "Regression_cos_phi": 0.0065009319223463535, "Regression_energy": 0.07168050110340118, "Total": 4.694797992706299, "Total_unweighted": 2.675158748868853}, "train_group": {}, "valid": {"Classification_binary": 3.8445844650268555, "Classification": 0.03223196044564247, "Regression_pt": 0.23559404909610748, "Regression_eta": 0.009108789265155792, "Regression_sin_phi": 0.008241827599704266, "Regression_cos_phi": 0.008364666253328323, "Regression_energy": 0.23418834805488586, "Total": 10.164624214172363, "Total_unweighted": 4.37231410574168, "metrics/particle/class_1/efficiency": 0.9133628606796265, "metrics/particle/class_1/pid_accuracy": 0.9763588905334473, "metrics/particle/class_2/efficiency": 0.28612059354782104, "metrics/particle/class_2/pid_accuracy": 0.5586481094360352, "metrics/particle/class_3/efficiency": 0.7251908183097839, "metrics/particle/class_3/pid_accuracy": 0.9794803261756897, "metrics/particle/class_4/efficiency": 0.8548895716667175, "metrics/particle/class_4/pid_accuracy": 0.25830256938934326, "metrics/particle/class_5/efficiency": 0.9519230723381042, "metrics/particle/class_5/pid_accuracy": 0.5252525210380554, "metrics/particle/count/bias_mean": 0.4359999895095825, "metrics/particle/count/mae": 4.8520002365112305, "metrics/particle/count/prediction_mean": 96.16400146484375, "metrics/particle/count/target_mean": 95.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.056938767433166504, "metrics/particle/event/energy_response_mean": 0.9893631339073181, "metrics/particle/event/met_abs_error": 9.040811538696289, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05538378655910492, "metrics/particle/event/scalar_pt_response_mean": 0.9933617115020752, "metrics/particle/event/vector_pt_closure": 0.06381924450397491, "metrics/particle/matched/delta_eta_abs_mean": 0.01853075809776783, "metrics/particle/matched/delta_phi_abs_mean": 0.017242809757590294, "metrics/particle/matched/delta_r_mean": 0.028195755556225777, "metrics/particle/matched/energy_relative_abs_mean": 0.09094478189945221, "metrics/particle/matched/pid_accuracy": 0.9513245820999146, "metrics/particle/matched/pt_relative_abs_mean": 0.08574598282575607, "metrics/particle/matching/duplicate_fraction": 0.0391414649784565, "metrics/particle/matching/efficiency": 0.785475492477417, "metrics/particle/matching/f1": 0.7836908102035522, "metrics/particle/matching/purity": 0.781914234161377, "metrics/particle/matching/target_coverage_dr0p05": 0.6642152667045593, "metrics/particle/matching/target_coverage_dr0p10": 0.8135968446731567, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7037439346313477, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.841300368309021}, "step_train_time": 0.49077391624450684, "step_valid_time": 3.0926866619847715, "step_total_time": 9135.834519863129, "elapsed_training_loop_seconds": 9135.834519863129, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9944452968864155, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14671553353582034, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1477, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1336, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1315, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9008766014834795, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9045362220717671, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9027027027027027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0954637779282329, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8867161159811193, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8903182125930941, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8885135135135135, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9008766014834795, "step_test_time": 4.881949672999326, "step_plot_time": 3.38660492998315, "step_validation_cycle_time": 11.808382118004374} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..6716e20ba9be85d85a6d156674b099cc1dbd2859 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_20mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..bf65afb2f254dab7c2ee50c6f8da9e51fd681640 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:021b3c4d471b514f4eb39716ca6cee795f603dd4d0d4135fc45266cc217d926a +size 4394 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b41f53afa4940e7b5784292e3d1745322b5013f3 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..9e07dc4e677d825e8c7a19f711281a5dc5139e3a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8699abac7ad7ee89def3fc5e299f632da02ac80d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..e4444ad577bb0b0dd49c6766623d17e75682705c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1b3bd904ef6e48a0cca6ec9fc2d7bba217b69804 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..f687c01897d2b9dc9b05250be054482a2a0ecf9f Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..95e22f25b5f232d7078450eb0d8f67e9ffee12e8 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..3e31e1ead1a979b02c2fb58554860e47ade69ed9 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..da32f2e2521e0cca05923fa220b68ed207e49d71 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..ba7b81a01e83b2a30322efae14389ac5d8213658 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..f1a7822f7666a7190e60c32d754f86fb184981df Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..a3cb3462d5d90aed046b212c715f7626f7d8231b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/plots_step_80000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/runs/train/events.out.tfevents.1794652427.workergpu161.3234262.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/runs/train/events.out.tfevents.1794652427.workergpu161.3234262.0 new file mode 100644 index 0000000000000000000000000000000000000000..ba4bb2ed165a7f761644453e66cebead6b3d4b86 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/runs/train/events.out.tfevents.1794652427.workergpu161.3234262.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af5b7dd90c40f84db2f39fdd4023ffd89eade9c7284b6cbe1651860cf4c91b19 +size 2953648 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/runs/valid/events.out.tfevents.1794652427.workergpu161.3234262.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/runs/valid/events.out.tfevents.1794652427.workergpu161.3234262.1 new file mode 100644 index 0000000000000000000000000000000000000000..00ddec0d6bef047988fc72b248bda5e06883886f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/runs/valid/events.out.tfevents.1794652427.workergpu161.3234262.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e32254aa1af5456857606dc63c62a8383822614f281d0b38da8023b22f77ee5 +size 4042848 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..ca72d47df93ef03894325258082f1f40a3951360 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_hcal_20mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_20mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_20mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_20mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41ced2bf7874403214e79b092102f7cede0dfa66 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_20mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/train.log new file mode 100644 index 0000000000000000000000000000000000000000..e3a22995ce20497ecc97e3482b5b741bc0049109 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/train.log @@ -0,0 +1,1353 @@ +rank0 [2026-11-14 05:33:44,673] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-14 05:33:44,673] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 05:33:45,194] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 05:33:45,194] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 05:33:45,197] INFO : Instantiating model +rank0 [2026-11-14 05:33:45,219] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 05:33:45,219] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 05:33:45,221] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 05:33:45,221] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 05:33:45,221] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 05:33:45,221] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 05:33:45,221] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,224] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 05:33:45,224] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,227] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 05:33:45,227] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,230] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 05:33:45,230] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,232] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 05:33:45,233] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,235] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 05:33:45,235] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,238] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 05:33:45,238] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 05:33:45,239] INFO : Initializing output DNNs +rank0 [2026-11-14 05:33:45,239] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,242] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,249] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 05:33:45,250] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 05:33:45,250] INFO : jagged_attention enabled=True +rank0 [2026-11-14 05:33:45,250] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 05:33:45,250] INFO : task_queries enabled=True +rank0 [2026-11-14 05:33:45,250] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 05:33:45,250] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 05:33:45,250] INFO : classification_query parameters: 256 +rank0 [2026-11-14 05:33:45,250] INFO : regression_query parameters: 256 +rank0 [2026-11-14 05:33:45,250] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 05:33:45,250] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 05:33:45,250] INFO : output_mode=elementwise +rank0 [2026-11-14 05:33:45,250] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 05:33:45,250] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 05:33:45,250] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 05:33:45,250] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 05:33:45,250] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 05:33:45,250] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 05:33:45,250] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 05:33:45,251] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 05:33:45,252] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 05:33:45,252] INFO : MLPF __init__ done +rank0 [2026-11-14 05:33:45,252] INFO : Instantiated model +rank0 [2026-11-14 05:33:45,252] INFO : Moving model to device rank=0 +rank0 [2026-11-14 05:33:45,261] INFO : Moved model to device rank=0 +rank0 [2026-11-14 05:33:45,262] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 05:33:45,360] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 05:33:45,360] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 05:33:45,361] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 05:33:45,361] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 05:33:45,361] INFO : Total parameters: 3910930 +rank0 [2026-11-14 05:33:45,364] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 05:33:45,367] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812 +rank0 [2026-11-14 05:33:45,367] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812 +rank0 [2026-11-14 05:33:45,367] INFO : Getting dataloaders +rank0 [2026-11-14 05:33:45,367] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:33:46,582] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,620] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,691] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,726] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,775] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,823] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,858] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,895] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,937] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,970] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,970] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 05:33:46,970] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 05:33:46,970] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 05:33:46,970] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:33:46,978] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:46,986] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:46,993] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,001] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,010] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,018] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,025] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,033] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,040] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,049] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,050] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 05:33:47,050] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 05:33:47,095] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:33:47,246] INFO : Got dataloaders +rank0 [2026-11-14 05:33:47,246] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 05:33:47,246] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 05:33:47,247] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 05:33:47,263] INFO : Creating train_iterator +rank0 [2026-11-14 05:33:47,264] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:33:47,291] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:33:47,436] INFO : Created train_iterator +rank0 [2026-11-14 05:34:00,297] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.106752347946167, 'Classification': 0.6368530941009521, 'Regression_pt': 0.5215235805511474, 'Regression_eta': 0.8829098019003868, 'Regression_sin_phi': 0.6576527345180512, 'Regression_cos_phi': 0.643730517923832, 'Regression_energy': 0.6670048692822457}, weights={'Classification_binary': 1.0, 'Classification': 11.159171104431152, 'Regression_pt': 13.62690544128418, 'Regression_eta': 8.049239158630371, 'Regression_sin_phi': 10.806238174438477, 'Regression_cos_phi': 11.039949417114258, 'Regression_energy': 10.654723167419434} +rank0 [2026-11-14 05:34:53,284] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 05:34:53,284] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:43,301] INFO : Step 1000/500000 rank0 | Train Loss: 11.1433 | Train Loss Unweighted: 5.0567 | Train Loss Components: Classification: 0.0634 | Classification_binary: 4.4926 | Regression_cos_phi: 0.0196 | Regression_energy: 0.2126 | Regression_eta: 0.0255 | Regression_pt: 0.2233 | Regression_sin_phi: 0.0197 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.7076 | Classification_binary: 4.4926 | Regression_cos_phi: 0.2167 | Regression_energy: 2.2653 | Regression_eta: 0.2053 | Regression_pt: 3.0431 | Regression_sin_phi: 0.2127 | LR: 2.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4830s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0344 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0628 | max_tensorboard_logging: 0.1388 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.4 | uss_MB: 1411.4 | child_rss_MB: 22481.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:35:56,611] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 05:35:56,612] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:00,710] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 05:37:00,710] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:37,200] INFO : Step 2000/500000 rank0 | Train Loss: 11.5924 | Train Loss Unweighted: 4.8458 | Train Loss Components: Classification: 0.0449 | Classification_binary: 4.2249 | Regression_cos_phi: 0.0192 | Regression_energy: 0.2507 | Regression_eta: 0.0251 | Regression_pt: 0.2638 | Regression_sin_phi: 0.0174 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.5006 | Classification_binary: 4.2249 | Regression_cos_phi: 0.2119 | Regression_energy: 2.6715 | Regression_eta: 0.2018 | Regression_pt: 3.5942 | Regression_sin_phi: 0.1876 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4633s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0342 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0592 | max_tensorboard_logging: 0.1267 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2572.2 | uss_MB: 1413.2 | child_rss_MB: 22483.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:37:37,202] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:38:04,350] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 05:38:04,350] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:06,585] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 05:39:06,586] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:28,573] INFO : Step 3000/500000 rank0 | Train Loss: 10.1246 | Train Loss Unweighted: 4.8538 | Train Loss Components: Classification: 0.0435 | Classification_binary: 4.3694 | Regression_cos_phi: 0.0160 | Regression_energy: 0.1972 | Regression_eta: 0.0128 | Regression_pt: 0.2010 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4855 | Classification_binary: 4.3694 | Regression_cos_phi: 0.1766 | Regression_energy: 2.1014 | Regression_eta: 0.1032 | Regression_pt: 2.7384 | Regression_sin_phi: 0.1500 | LR: 2.05e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5311s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0438 | max_loss: 0.0031 | max_backward: 0.0410 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1454 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2575.9 | uss_MB: 1417.4 | child_rss_MB: 22482.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:40:10,379] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 05:40:10,379] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:13,644] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 05:41:13,644] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:21,476] INFO : Step 4000/500000 rank0 | Train Loss: 10.5804 | Train Loss Unweighted: 4.8436 | Train Loss Components: Classification: 0.0411 | Classification_binary: 4.3151 | Regression_cos_phi: 0.0191 | Regression_energy: 0.2207 | Regression_eta: 0.0171 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4586 | Classification_binary: 4.3151 | Regression_cos_phi: 0.2105 | Regression_energy: 2.3515 | Regression_eta: 0.1380 | Regression_pt: 2.9751 | Regression_sin_phi: 0.1317 | LR: 2.08e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.5266s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0365 | max_loss: 0.0030 | max_backward: 0.0426 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1491 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2576.2 | uss_MB: 1417.7 | child_rss_MB: 22485.2 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 05:41:21,478] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:42:17,292] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 05:42:17,292] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:43:14,483] INFO : Step 5000/500000 rank0 | Train Loss: 9.9829 | Train Loss Unweighted: 4.4750 | Train Loss Components: Classification: 0.0471 | Classification_binary: 3.9661 | Regression_cos_phi: 0.0110 | Regression_energy: 0.2068 | Regression_eta: 0.0227 | Regression_pt: 0.2102 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.5253 | Classification_binary: 3.9661 | Regression_cos_phi: 0.1211 | Regression_energy: 2.2037 | Regression_eta: 0.1827 | Regression_pt: 2.8649 | Regression_sin_phi: 0.1191 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4882s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0592 | max_tensorboard_logging: 0.1393 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2575.9 | uss_MB: 1417.4 | child_rss_MB: 22485.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:43:20,979] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 05:43:20,979] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:24,058] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 05:44:24,058] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:45:07,054] INFO : Step 6000/500000 rank0 | Train Loss: 9.5371 | Train Loss Unweighted: 4.4290 | Train Loss Components: Classification: 0.0303 | Classification_binary: 3.9625 | Regression_cos_phi: 0.0077 | Regression_energy: 0.2016 | Regression_eta: 0.0100 | Regression_pt: 0.2056 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3378 | Classification_binary: 3.9625 | Regression_cos_phi: 0.0854 | Regression_energy: 2.1478 | Regression_eta: 0.0809 | Regression_pt: 2.8014 | Regression_sin_phi: 0.1214 | LR: 2.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4766s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0357 | max_loss: 0.0027 | max_backward: 0.0391 | max_optimizer_step: 0.0616 | max_tensorboard_logging: 0.1184 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2576.2 | uss_MB: 1417.7 | child_rss_MB: 22486.4 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 7.9 +rank0 [2026-11-14 05:45:07,056] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:45:27,674] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 05:45:27,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:30,754] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 05:46:30,755] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:59,976] INFO : Step 7000/500000 rank0 | Train Loss: 9.5948 | Train Loss Unweighted: 4.4012 | Train Loss Components: Classification: 0.0388 | Classification_binary: 3.9239 | Regression_cos_phi: 0.0165 | Regression_energy: 0.1925 | Regression_eta: 0.0129 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0178 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4328 | Classification_binary: 3.9239 | Regression_cos_phi: 0.1820 | Regression_energy: 2.0506 | Regression_eta: 0.1042 | Regression_pt: 2.7093 | Regression_sin_phi: 0.1919 | LR: 2.26e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5081s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0372 | max_loss: 0.0029 | max_backward: 0.0419 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1233 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2576.0 | uss_MB: 1417.5 | child_rss_MB: 22487.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:47:34,934] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 05:47:34,935] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:38,181] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 05:48:38,181] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:53,445] INFO : Step 8000/500000 rank0 | Train Loss: 9.4272 | Train Loss Unweighted: 4.3602 | Train Loss Components: Classification: 0.0430 | Classification_binary: 3.8964 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1993 | Regression_eta: 0.0074 | Regression_pt: 0.1959 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4802 | Classification_binary: 3.8964 | Regression_cos_phi: 0.0956 | Regression_energy: 2.1238 | Regression_eta: 0.0595 | Regression_pt: 2.6692 | Regression_sin_phi: 0.1024 | LR: 2.34e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5173s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0370 | max_loss: 0.0029 | max_backward: 0.0413 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1346 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2576.1 | uss_MB: 1417.6 | child_rss_MB: 22487.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 28.3 +rank0 [2026-11-14 05:48:53,447] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:48:53,518] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:49:41,783] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 05:49:41,784] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:45,084] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 05:50:45,084] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:46,390] INFO : Step 9000/500000 rank0 | Train Loss: 9.7958 | Train Loss Unweighted: 4.5659 | Train Loss Components: Classification: 0.0366 | Classification_binary: 4.0830 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2002 | Regression_eta: 0.0194 | Regression_pt: 0.1997 | Regression_sin_phi: 0.0164 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4087 | Classification_binary: 4.0830 | Regression_cos_phi: 0.1168 | Regression_energy: 2.1332 | Regression_eta: 0.1563 | Regression_pt: 2.7207 | Regression_sin_phi: 0.1770 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4892s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0029 | max_backward: 0.0407 | max_optimizer_step: 0.0645 | max_tensorboard_logging: 0.1206 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2573.9 | uss_MB: 1415.3 | child_rss_MB: 22485.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 05:51:48,820] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 05:51:48,821] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:39,797] INFO : Step 10000/500000 rank0 | Train Loss: 9.9879 | Train Loss Unweighted: 4.3750 | Train Loss Components: Classification: 0.0329 | Classification_binary: 3.8631 | Regression_cos_phi: 0.0108 | Regression_energy: 0.2252 | Regression_eta: 0.0081 | Regression_pt: 0.2253 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3669 | Classification_binary: 3.8631 | Regression_cos_phi: 0.1191 | Regression_energy: 2.3995 | Regression_eta: 0.0654 | Regression_pt: 3.0706 | Regression_sin_phi: 0.1032 | LR: 2.52e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.5194s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0385 | max_loss: 0.0032 | max_backward: 0.0411 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2574.4 | uss_MB: 1415.8 | child_rss_MB: 22488.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:52:39,799] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:52:39,869] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:52:39,870] INFO : Running validation on rank0 +rank0 [2026-11-14 05:52:39,870] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:52:39,870] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:52:40,088] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 05:52:40,284] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:52:40,368] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:52:40,425] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:52:40,482] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:52:40,529] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:52:40,583] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:52:40,638] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:52:40,688] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:52:40,766] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:52:40,846] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:52:40,901] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:52:40,960] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:52:41,010] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:52:41,068] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:52:41,124] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:52:41,179] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:52:41,184] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:52:41,185] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:52:41,188] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:52:41,190] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:52:41,191] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:52:41,195] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:52:41,197] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:52:41,198] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:52:41,201] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:52:41,201] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:52:41,203] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:52:41,207] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:52:41,209] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:52:41,211] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:52:41,213] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:52:42,806] INFO : VALIDATION | Step=10000/500000 | Train Loss=9.9879 | Valid Loss=9.7571 | Train Loss Unweighted=4.3750 | Valid Loss Unweighted=4.3510 | Stale=0 | ETA=927.4m +rank0 [2026-11-14 05:52:42,807] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:52:43,333] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:52:43,341] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:43,391] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:52:44,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:52:44,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:52:44,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:52:44,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:52:45,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:52:45,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:52:45,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:52:45,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:52:46,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:52:46,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:52:46,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:52:46,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:52:47,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:52:47,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:52:47,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:52:47,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:52:48,039] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:52:48,046] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:48,647] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:49,408] INFO : Plotted number of elements +rank0 [2026-11-14 05:52:50,846] INFO : Plotted jets +rank0 [2026-11-14 05:52:51,850] INFO : Plotted jet ratio +rank0 [2026-11-14 05:53:05,480] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 05:53:05,480] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:09,474] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 05:54:09,474] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:46,409] INFO : Step 11000/500000 rank0 | Train Loss: 9.1829 | Train Loss Unweighted: 4.2780 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.8318 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1946 | Regression_eta: 0.0074 | Regression_pt: 0.2013 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3331 | Classification_binary: 3.8318 | Regression_cos_phi: 0.0593 | Regression_energy: 2.0735 | Regression_eta: 0.0597 | Regression_pt: 2.7427 | Regression_sin_phi: 0.0828 | LR: 2.63e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5106s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.1278 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2749.4 | uss_MB: 1605.7 | child_rss_MB: 22490.5 | allocated_MB: 125.7 | reserved_MB: 620.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:55:13,517] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 05:55:13,517] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:16,956] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 05:56:16,956] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:39,628] INFO : Step 12000/500000 rank0 | Train Loss: 7.7118 | Train Loss Unweighted: 3.7300 | Train Loss Components: Classification: 0.0281 | Classification_binary: 3.3671 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1562 | Regression_eta: 0.0053 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3138 | Classification_binary: 3.3671 | Regression_cos_phi: 0.0678 | Regression_energy: 1.6641 | Regression_eta: 0.0423 | Regression_pt: 2.1723 | Regression_sin_phi: 0.0844 | LR: 2.75e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5104s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0031 | max_backward: 0.0401 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1245 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2749.6 | uss_MB: 1605.9 | child_rss_MB: 22490.6 | allocated_MB: 126.1 | reserved_MB: 640.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:56:39,630] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:56:39,704] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:57:20,636] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 05:57:20,636] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:24,102] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 05:58:24,102] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:32,890] INFO : Step 13000/500000 rank0 | Train Loss: 9.2807 | Train Loss Unweighted: 4.3454 | Train Loss Components: Classification: 0.0326 | Classification_binary: 3.8936 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1948 | Regression_eta: 0.0090 | Regression_pt: 0.1935 | Regression_sin_phi: 0.0132 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3635 | Classification_binary: 3.8936 | Regression_cos_phi: 0.0965 | Regression_energy: 2.0751 | Regression_eta: 0.0721 | Regression_pt: 2.6371 | Regression_sin_phi: 0.1428 | LR: 2.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5251s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0384 | max_loss: 0.0031 | max_backward: 0.0420 | max_optimizer_step: 0.0687 | max_tensorboard_logging: 0.1273 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2676.7 | uss_MB: 1533.0 | child_rss_MB: 22489.8 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:59:27,674] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:59:27,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:25,938] INFO : Step 14000/500000 rank0 | Train Loss: 9.3603 | Train Loss Unweighted: 4.2548 | Train Loss Components: Classification: 0.0337 | Classification_binary: 3.7887 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1869 | Regression_eta: 0.0162 | Regression_pt: 0.2107 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3766 | Classification_binary: 3.7887 | Regression_cos_phi: 0.0815 | Regression_energy: 1.9909 | Regression_eta: 0.1305 | Regression_pt: 2.8710 | Regression_sin_phi: 0.1211 | LR: 3.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5271s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0028 | max_backward: 0.0420 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.1227 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2676.9 | uss_MB: 1533.2 | child_rss_MB: 22489.5 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:00:25,940] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:00:26,012] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 06:00:31,644] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 06:00:31,644] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:34,883] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 06:01:34,883] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:19,385] INFO : Step 15000/500000 rank0 | Train Loss: 9.0935 | Train Loss Unweighted: 4.2235 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.7785 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1883 | Regression_eta: 0.0111 | Regression_pt: 0.1956 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3610 | Classification_binary: 3.7785 | Regression_cos_phi: 0.1006 | Regression_energy: 2.0061 | Regression_eta: 0.0892 | Regression_pt: 2.6649 | Regression_sin_phi: 0.0932 | LR: 3.17e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5162s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0375 | max_loss: 0.0032 | max_backward: 0.0420 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2679.9 | uss_MB: 1536.2 | child_rss_MB: 22491.6 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:02:38,974] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 06:02:38,974] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:42,612] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 06:03:42,612] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:11,862] INFO : Step 16000/500000 rank0 | Train Loss: 9.2886 | Train Loss Unweighted: 4.2352 | Train Loss Components: Classification: 0.0360 | Classification_binary: 3.7714 | Regression_cos_phi: 0.0138 | Regression_energy: 0.1938 | Regression_eta: 0.0131 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4019 | Classification_binary: 3.7714 | Regression_cos_phi: 0.1527 | Regression_energy: 2.0647 | Regression_eta: 0.1050 | Regression_pt: 2.6795 | Regression_sin_phi: 0.1134 | LR: 3.33e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4922s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0350 | max_loss: 0.0027 | max_backward: 0.0390 | max_optimizer_step: 0.0624 | max_tensorboard_logging: 0.1275 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2680.3 | uss_MB: 1536.6 | child_rss_MB: 22490.8 | allocated_MB: 126.1 | reserved_MB: 640.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 06:04:11,866] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:04:11,936] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 06:04:45,452] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 06:04:45,452] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:50,591] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 06:05:50,591] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:06:06,257] INFO : Step 17000/500000 rank0 | Train Loss: 9.3975 | Train Loss Unweighted: 4.2566 | Train Loss Components: Classification: 0.0354 | Classification_binary: 3.7848 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1962 | Regression_eta: 0.0172 | Regression_pt: 0.2043 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3945 | Classification_binary: 3.7848 | Regression_cos_phi: 0.1189 | Regression_energy: 2.0903 | Regression_eta: 0.1385 | Regression_pt: 2.7840 | Regression_sin_phi: 0.0865 | LR: 3.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4665s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0340 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0624 | max_tensorboard_logging: 0.1217 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2676.1 | uss_MB: 1532.5 | child_rss_MB: 22493.2 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:52,400] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 06:06:52,401] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:56,367] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 06:07:56,367] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:58,588] INFO : Step 18000/500000 rank0 | Train Loss: 9.0548 | Train Loss Unweighted: 4.1751 | Train Loss Components: Classification: 0.0318 | Classification_binary: 3.7275 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1821 | Regression_eta: 0.0181 | Regression_pt: 0.1968 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3549 | Classification_binary: 3.7275 | Regression_cos_phi: 0.0976 | Regression_energy: 1.9406 | Regression_eta: 0.1456 | Regression_pt: 2.6813 | Regression_sin_phi: 0.1074 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.5173s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0031 | max_backward: 0.0449 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1339 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2677.0 | uss_MB: 1533.3 | child_rss_MB: 22491.3 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:07:58,591] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:07:58,661] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 06:09:00,429] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 06:09:00,430] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:52,035] INFO : Step 19000/500000 rank0 | Train Loss: 8.4928 | Train Loss Unweighted: 4.1407 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.7439 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1672 | Regression_eta: 0.0061 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3347 | Classification_binary: 3.7439 | Regression_cos_phi: 0.1183 | Regression_energy: 1.7813 | Regression_eta: 0.0490 | Regression_pt: 2.3701 | Regression_sin_phi: 0.0955 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5150s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0030 | max_backward: 0.0413 | max_optimizer_step: 0.0751 | max_tensorboard_logging: 0.1210 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2679.9 | uss_MB: 1536.2 | child_rss_MB: 22492.1 | allocated_MB: 125.7 | reserved_MB: 642.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:10:04,267] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 06:10:04,267] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:07,551] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 06:11:07,551] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:47,234] INFO : Step 20000/500000 rank0 | Train Loss: 9.9339 | Train Loss Unweighted: 4.5901 | Train Loss Components: Classification: 0.0321 | Classification_binary: 4.1021 | Regression_cos_phi: 0.0103 | Regression_energy: 0.2105 | Regression_eta: 0.0102 | Regression_pt: 0.2145 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3581 | Classification_binary: 4.1021 | Regression_cos_phi: 0.1141 | Regression_energy: 2.2423 | Regression_eta: 0.0821 | Regression_pt: 2.9224 | Regression_sin_phi: 0.1129 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5597s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0032 | max_backward: 0.0417 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.1712 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2686.8 | uss_MB: 1543.2 | child_rss_MB: 22491.2 | allocated_MB: 125.7 | reserved_MB: 642.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:11:47,268] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:11:47,385] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:11:47,402] INFO : Running validation on rank0 +rank0 [2026-11-14 06:11:47,402] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:11:47,402] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:11:47,583] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 06:11:47,709] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:11:47,857] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:11:47,929] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:11:47,999] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:11:48,067] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:11:48,137] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:11:48,204] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:11:48,318] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:11:48,436] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:11:48,553] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:11:48,621] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:11:48,687] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:11:48,753] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:11:48,820] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:11:48,887] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:11:49,003] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:11:49,010] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:11:49,012] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:11:49,015] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:11:49,017] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:11:49,019] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:11:49,021] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:11:49,023] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:11:49,025] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:11:49,027] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:11:49,029] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:11:49,030] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:11:49,034] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:11:49,036] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:11:49,038] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:11:49,040] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:11:50,653] INFO : VALIDATION | Step=20000/500000 | Train Loss=9.9339 | Valid Loss=9.4105 | Train Loss Unweighted=4.5901 | Valid Loss Unweighted=4.1819 | Stale=0 | ETA=913.4m +rank0 [2026-11-14 06:11:50,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:11:51,086] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:11:51,117] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:51,217] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:11:51,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:11:51,944] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:11:52,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:11:52,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:11:52,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:11:52,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:11:53,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:11:53,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:11:53,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:11:54,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:11:54,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:11:54,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:11:54,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:11:55,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:11:55,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:11:55,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:11:55,573] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:11:55,579] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:56,165] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:56,564] INFO : Plotted number of elements +rank0 [2026-11-14 06:11:57,923] INFO : Plotted jets +rank0 [2026-11-14 06:11:58,930] INFO : Plotted jet ratio +rank0 [2026-11-14 06:12:25,574] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 06:12:25,575] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:28,586] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 06:13:28,586] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:51,988] INFO : Step 21000/500000 rank0 | Train Loss: 8.8431 | Train Loss Unweighted: 4.0429 | Train Loss Components: Classification: 0.0293 | Classification_binary: 3.6059 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1924 | Regression_eta: 0.0073 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3275 | Classification_binary: 3.6059 | Regression_cos_phi: 0.0670 | Regression_energy: 2.0504 | Regression_eta: 0.0584 | Regression_pt: 2.6706 | Regression_sin_phi: 0.0632 | LR: 4.28e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5283s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0371 | max_loss: 0.0032 | max_backward: 0.0404 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1384 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2722.1 | uss_MB: 1578.5 | child_rss_MB: 22493.1 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:14:31,580] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 06:14:31,580] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:34,294] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 06:15:34,295] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:43,914] INFO : Step 22000/500000 rank0 | Train Loss: 7.6051 | Train Loss Unweighted: 3.9735 | Train Loss Components: Classification: 0.0332 | Classification_binary: 3.6386 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1375 | Regression_eta: 0.0109 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3703 | Classification_binary: 3.6386 | Regression_cos_phi: 0.0937 | Regression_energy: 1.4649 | Regression_eta: 0.0881 | Regression_pt: 1.8596 | Regression_sin_phi: 0.0900 | LR: 4.50e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5138s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0413 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1289 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2734.3 | uss_MB: 1590.6 | child_rss_MB: 22493.7 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:15:43,916] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:15:43,989] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:16:37,976] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 06:16:37,976] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:36,210] INFO : Step 23000/500000 rank0 | Train Loss: 8.4608 | Train Loss Unweighted: 4.1721 | Train Loss Components: Classification: 0.0333 | Classification_binary: 3.7811 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1685 | Regression_eta: 0.0060 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3715 | Classification_binary: 3.7811 | Regression_cos_phi: 0.0682 | Regression_energy: 1.7955 | Regression_eta: 0.0486 | Regression_pt: 2.3325 | Regression_sin_phi: 0.0634 | LR: 4.73e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4880s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0031 | max_backward: 0.0400 | max_optimizer_step: 0.0660 | max_tensorboard_logging: 0.1162 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2738.2 | uss_MB: 1619.9 | child_rss_MB: 22494.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 38.3 +rank0 [2026-11-14 06:17:40,897] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 06:17:40,897] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:43,223] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 06:18:43,223] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:19:27,541] INFO : Step 24000/500000 rank0 | Train Loss: 8.4071 | Train Loss Unweighted: 3.9444 | Train Loss Components: Classification: 0.0358 | Classification_binary: 3.5361 | Regression_cos_phi: 0.0148 | Regression_energy: 0.1705 | Regression_eta: 0.0069 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3992 | Classification_binary: 3.5361 | Regression_cos_phi: 0.1631 | Regression_energy: 1.8167 | Regression_eta: 0.0557 | Regression_pt: 2.3568 | Regression_sin_phi: 0.0794 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4828s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0027 | max_backward: 0.0394 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.1199 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2741.0 | uss_MB: 1646.7 | child_rss_MB: 22495.4 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 14.3 +rank0 [2026-11-14 06:19:27,543] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:19:27,616] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:19:46,064] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 06:19:46,064] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:48,693] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 06:20:48,693] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:21:19,615] INFO : Step 25000/500000 rank0 | Train Loss: 8.1063 | Train Loss Unweighted: 4.1059 | Train Loss Components: Classification: 0.0367 | Classification_binary: 3.7373 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1516 | Regression_eta: 0.0120 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4098 | Classification_binary: 3.7373 | Regression_cos_phi: 0.0951 | Regression_energy: 1.6153 | Regression_eta: 0.0963 | Regression_pt: 2.0635 | Regression_sin_phi: 0.0890 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4967s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0366 | max_loss: 0.0030 | max_backward: 0.0401 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1218 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2737.2 | uss_MB: 1651.6 | child_rss_MB: 22492.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:21:51,782] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 06:21:51,782] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:54,120] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 06:22:54,121] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:11,151] INFO : Step 26000/500000 rank0 | Train Loss: 8.5651 | Train Loss Unweighted: 3.9457 | Train Loss Components: Classification: 0.0366 | Classification_binary: 3.5231 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1812 | Regression_eta: 0.0080 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4090 | Classification_binary: 3.5231 | Regression_cos_phi: 0.0928 | Regression_energy: 1.9308 | Regression_eta: 0.0647 | Regression_pt: 2.4623 | Regression_sin_phi: 0.0825 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5038s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0402 | max_optimizer_step: 0.0654 | max_tensorboard_logging: 0.1339 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2737.9 | uss_MB: 1652.3 | child_rss_MB: 22495.8 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:23:11,153] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:23:11,248] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:23:57,559] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 06:23:57,559] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:00,155] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 06:25:00,155] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:03,199] INFO : Step 27000/500000 rank0 | Train Loss: 10.6380 | Train Loss Unweighted: 4.4254 | Train Loss Components: Classification: 0.0389 | Classification_binary: 3.8600 | Regression_cos_phi: 0.0075 | Regression_energy: 0.2456 | Regression_eta: 0.0095 | Regression_pt: 0.2539 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4342 | Classification_binary: 3.8600 | Regression_cos_phi: 0.0826 | Regression_energy: 2.6164 | Regression_eta: 0.0767 | Regression_pt: 3.4593 | Regression_sin_phi: 0.1089 | LR: 5.74e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4811s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0394 | max_optimizer_step: 0.0635 | max_tensorboard_logging: 0.1207 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2740.9 | uss_MB: 1655.3 | child_rss_MB: 22495.0 | allocated_MB: 125.7 | reserved_MB: 634.0 | inactive_split_MB: 14.3 +rank0 [2026-11-14 06:26:03,298] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 06:26:03,298] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:54,965] INFO : Step 28000/500000 rank0 | Train Loss: 9.0425 | Train Loss Unweighted: 4.1518 | Train Loss Components: Classification: 0.0347 | Classification_binary: 3.7036 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1881 | Regression_eta: 0.0126 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3876 | Classification_binary: 3.7036 | Regression_cos_phi: 0.1212 | Regression_energy: 2.0043 | Regression_eta: 0.1012 | Regression_pt: 2.6271 | Regression_sin_phi: 0.0973 | LR: 6.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5167s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1233 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2741.2 | uss_MB: 1655.6 | child_rss_MB: 22495.1 | allocated_MB: 125.7 | reserved_MB: 634.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:26:54,967] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:26:55,045] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:27:06,174] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 06:27:06,174] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:08,513] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 06:28:08,513] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:46,205] INFO : Step 29000/500000 rank0 | Train Loss: 8.8674 | Train Loss Unweighted: 4.0665 | Train Loss Components: Classification: 0.0313 | Classification_binary: 3.6289 | Regression_cos_phi: 0.0125 | Regression_energy: 0.1834 | Regression_eta: 0.0075 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3493 | Classification_binary: 3.6289 | Regression_cos_phi: 0.1384 | Regression_energy: 1.9541 | Regression_eta: 0.0607 | Regression_pt: 2.6268 | Regression_sin_phi: 0.1092 | LR: 6.29e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4878s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0031 | max_backward: 0.0399 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1208 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2741.3 | uss_MB: 1664.4 | child_rss_MB: 22494.6 | allocated_MB: 125.7 | reserved_MB: 634.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:29:10,869] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 06:29:10,869] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:13,020] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 06:30:13,020] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:37,032] INFO : Step 30000/500000 rank0 | Train Loss: 8.2845 | Train Loss Unweighted: 4.0901 | Train Loss Components: Classification: 0.0393 | Classification_binary: 3.7052 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1591 | Regression_eta: 0.0093 | Regression_pt: 0.1612 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4389 | Classification_binary: 3.7052 | Regression_cos_phi: 0.0775 | Regression_energy: 1.6950 | Regression_eta: 0.0752 | Regression_pt: 2.1971 | Regression_sin_phi: 0.0956 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4910s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0386 | max_loss: 0.0029 | max_backward: 0.0414 | max_optimizer_step: 0.0648 | max_tensorboard_logging: 0.1230 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2738.4 | uss_MB: 1665.6 | child_rss_MB: 22496.2 | allocated_MB: 126.1 | reserved_MB: 634.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:30:37,034] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:30:37,104] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:30:37,105] INFO : Running validation on rank0 +rank0 [2026-11-14 06:30:37,105] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:30:37,105] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:30:37,259] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 06:30:37,377] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:30:37,492] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:30:37,561] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:30:37,629] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:30:37,696] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:30:37,762] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:30:37,828] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:30:37,944] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:30:38,063] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:30:38,178] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:30:38,245] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:30:38,313] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:30:38,379] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:30:38,447] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:30:38,512] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:30:38,625] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:30:38,632] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:30:38,634] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:30:38,637] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:30:38,639] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:30:38,641] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:30:38,643] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:30:38,645] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:30:38,647] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:30:38,649] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:30:38,651] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:30:38,653] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:30:38,655] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:30:38,657] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:30:38,659] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:30:38,661] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:30:40,236] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.2845 | Valid Loss=8.9936 | Train Loss Unweighted=4.0901 | Valid Loss Unweighted=4.0514 | Stale=0 | ETA=891.2m +rank0 [2026-11-14 06:30:40,236] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:30:40,591] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:30:40,600] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:40,645] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:30:41,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:30:41,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:30:41,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:30:41,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:30:42,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:30:42,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:30:42,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:30:42,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:30:43,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:30:43,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:30:43,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:30:44,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:30:44,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:30:44,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:30:44,770] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:30:45,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:30:45,053] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:30:45,061] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:45,644] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:46,030] INFO : Plotted number of elements +rank0 [2026-11-14 06:30:47,429] INFO : Plotted jets +rank0 [2026-11-14 06:30:48,487] INFO : Plotted jet ratio +rank0 [2026-11-14 06:31:28,087] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 06:31:28,087] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:29,843] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 06:32:29,843] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:39,833] INFO : Step 31000/500000 rank0 | Train Loss: 8.7836 | Train Loss Unweighted: 3.9801 | Train Loss Components: Classification: 0.0326 | Classification_binary: 3.5394 | Regression_cos_phi: 0.0143 | Regression_energy: 0.1814 | Regression_eta: 0.0112 | Regression_pt: 0.1859 | Regression_sin_phi: 0.0154 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3642 | Classification_binary: 3.5394 | Regression_cos_phi: 0.1575 | Regression_energy: 1.9323 | Regression_eta: 0.0900 | Regression_pt: 2.5338 | Regression_sin_phi: 0.1663 | LR: 6.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4570s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0370 | max_loss: 0.0031 | max_backward: 0.0394 | max_optimizer_step: 0.0596 | max_tensorboard_logging: 0.1216 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2897.6 | uss_MB: 1830.2 | child_rss_MB: 22491.6 | allocated_MB: 126.1 | reserved_MB: 628.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:33:34,951] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 06:33:34,952] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:34,417] INFO : Step 32000/500000 rank0 | Train Loss: 10.0047 | Train Loss Unweighted: 4.1107 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0057 | Regression_energy: 0.2440 | Regression_eta: 0.0092 | Regression_pt: 0.2445 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3105 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0628 | Regression_energy: 2.5993 | Regression_eta: 0.0742 | Regression_pt: 3.3314 | Regression_sin_phi: 0.0518 | LR: 7.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4976s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0031 | max_backward: 0.0407 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1207 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2906.3 | uss_MB: 1838.8 | child_rss_MB: 22490.7 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:34:34,419] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:34:34,490] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:34:38,164] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 06:34:38,164] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:41,083] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 06:35:41,083] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:26,994] INFO : Step 33000/500000 rank0 | Train Loss: 9.2693 | Train Loss Unweighted: 3.7727 | Train Loss Components: Classification: 0.0347 | Classification_binary: 3.2731 | Regression_cos_phi: 0.0056 | Regression_energy: 0.2225 | Regression_eta: 0.0047 | Regression_pt: 0.2238 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3872 | Classification_binary: 3.2731 | Regression_cos_phi: 0.0614 | Regression_energy: 2.3712 | Regression_eta: 0.0380 | Regression_pt: 3.0493 | Regression_sin_phi: 0.0890 | LR: 7.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5011s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0366 | max_loss: 0.0028 | max_backward: 0.0406 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1320 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.6 | uss_MB: 1763.2 | child_rss_MB: 22491.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:36:44,797] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 06:36:44,797] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:48,216] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 06:37:48,216] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:20,003] INFO : Step 34000/500000 rank0 | Train Loss: 9.1484 | Train Loss Unweighted: 4.0477 | Train Loss Components: Classification: 0.0357 | Classification_binary: 3.5836 | Regression_cos_phi: 0.0124 | Regression_energy: 0.1882 | Regression_eta: 0.0091 | Regression_pt: 0.2083 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3986 | Classification_binary: 3.5836 | Regression_cos_phi: 0.1371 | Regression_energy: 2.0049 | Regression_eta: 0.0730 | Regression_pt: 2.8388 | Regression_sin_phi: 0.1125 | LR: 7.83e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4847s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0390 | max_loss: 0.0027 | max_backward: 0.0410 | max_optimizer_step: 0.0655 | max_tensorboard_logging: 0.1230 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2849.7 | uss_MB: 1782.2 | child_rss_MB: 22492.9 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:38:20,005] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:38:20,075] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:38:51,650] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 06:38:51,650] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:39:54,603] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 06:39:54,604] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:12,751] INFO : Step 35000/500000 rank0 | Train Loss: 8.5985 | Train Loss Unweighted: 4.0657 | Train Loss Components: Classification: 0.0345 | Classification_binary: 3.6470 | Regression_cos_phi: 0.0134 | Regression_energy: 0.1771 | Regression_eta: 0.0150 | Regression_pt: 0.1699 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3855 | Classification_binary: 3.6470 | Regression_cos_phi: 0.1476 | Regression_energy: 1.8873 | Regression_eta: 0.1206 | Regression_pt: 2.3149 | Regression_sin_phi: 0.0957 | LR: 8.16e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4963s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0409 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1246 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2853.9 | uss_MB: 1786.4 | child_rss_MB: 22493.1 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:40:58,085] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 06:40:58,085] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:01,108] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 06:42:01,109] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:05,088] INFO : Step 36000/500000 rank0 | Train Loss: 6.5200 | Train Loss Unweighted: 3.3483 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.0579 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1220 | Regression_eta: 0.0058 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3143 | Classification_binary: 3.0579 | Regression_cos_phi: 0.0499 | Regression_energy: 1.2998 | Regression_eta: 0.0469 | Regression_pt: 1.6782 | Regression_sin_phi: 0.0729 | LR: 8.50e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5009s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0033 | max_backward: 0.0411 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1224 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2853.9 | uss_MB: 1786.5 | child_rss_MB: 22491.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:42:05,090] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:42:05,159] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:43:04,337] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 06:43:04,337] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:43:57,351] INFO : Step 37000/500000 rank0 | Train Loss: 8.4942 | Train Loss Unweighted: 3.7323 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.2968 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1807 | Regression_eta: 0.0107 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3518 | Classification_binary: 3.2968 | Regression_cos_phi: 0.1071 | Regression_energy: 1.9248 | Regression_eta: 0.0858 | Regression_pt: 2.5812 | Regression_sin_phi: 0.1467 | LR: 8.85e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4784s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0027 | max_backward: 0.0403 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1169 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2854.0 | uss_MB: 1786.5 | child_rss_MB: 22491.9 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:44:07,646] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 06:44:07,646] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:45:11,086] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 06:45:11,086] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:45:49,940] INFO : Step 38000/500000 rank0 | Train Loss: 8.2303 | Train Loss Unweighted: 3.8784 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.4812 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1722 | Regression_eta: 0.0067 | Regression_pt: 0.1734 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3159 | Classification_binary: 3.4812 | Regression_cos_phi: 0.0959 | Regression_energy: 1.8350 | Regression_eta: 0.0543 | Regression_pt: 2.3633 | Regression_sin_phi: 0.0847 | LR: 9.21e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4964s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0032 | max_backward: 0.0415 | max_optimizer_step: 0.0642 | max_tensorboard_logging: 0.1245 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2851.5 | uss_MB: 1784.0 | child_rss_MB: 22493.4 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:45:49,942] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:45:50,014] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:46:14,148] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 06:46:14,148] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:16,885] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 06:47:16,885] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:42,229] INFO : Step 39000/500000 rank0 | Train Loss: 8.7446 | Train Loss Unweighted: 3.9702 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.5341 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1846 | Regression_eta: 0.0087 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0132 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3194 | Classification_binary: 3.5341 | Regression_cos_phi: 0.1153 | Regression_energy: 1.9665 | Regression_eta: 0.0703 | Regression_pt: 2.5967 | Regression_sin_phi: 0.1421 | LR: 9.57e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4976s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0412 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.1233 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2854.6 | uss_MB: 1787.2 | child_rss_MB: 22499.7 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:48:20,077] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 06:48:20,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:22,873] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 06:49:22,873] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:34,191] INFO : Step 40000/500000 rank0 | Train Loss: 7.5032 | Train Loss Unweighted: 3.5061 | Train Loss Components: Classification: 0.0201 | Classification_binary: 3.1409 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1564 | Regression_eta: 0.0082 | Regression_pt: 0.1602 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2244 | Classification_binary: 3.1409 | Regression_cos_phi: 0.1183 | Regression_energy: 1.6661 | Regression_eta: 0.0660 | Regression_pt: 2.1828 | Regression_sin_phi: 0.1048 | LR: 9.94e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4891s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0407 | max_optimizer_step: 0.0648 | max_tensorboard_logging: 0.1216 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.0 | uss_MB: 1787.6 | child_rss_MB: 22498.8 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:49:34,193] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 06:49:34,265] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:49:34,266] INFO : Running validation on rank0 +rank0 [2026-11-14 06:49:34,266] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:49:34,266] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:49:34,416] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 06:49:34,536] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:49:34,655] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:49:34,727] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:49:34,795] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:49:34,863] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:49:34,933] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:49:35,003] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:49:35,124] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:49:35,244] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:49:35,361] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:49:35,430] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:49:35,497] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:49:35,566] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:49:35,633] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:49:35,701] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:49:35,820] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:49:35,828] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:49:35,830] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:49:35,832] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:49:35,837] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:49:35,839] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:49:35,841] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:49:35,843] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:49:35,844] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:49:35,847] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:49:35,849] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:49:35,851] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:49:35,853] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:49:35,855] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:49:35,857] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:49:35,859] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:49:37,445] INFO : VALIDATION | Step=40000/500000 | Train Loss=7.5032 | Valid Loss=8.5265 | Train Loss Unweighted=3.5061 | Valid Loss Unweighted=3.8310 | Stale=0 | ETA=872.1m +rank0 [2026-11-14 06:49:37,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:49:38,146] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:49:38,154] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:38,197] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:49:38,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:49:38,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:49:39,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:49:39,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:49:39,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:49:40,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:49:40,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:49:40,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:49:40,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:49:41,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:49:41,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:49:41,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:49:41,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:49:42,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:49:42,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:49:42,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:49:42,616] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:49:42,634] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:43,208] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:43,593] INFO : Plotted number of elements +rank0 [2026-11-14 06:49:44,978] INFO : Plotted jets +rank0 [2026-11-14 06:49:45,992] INFO : Plotted jet ratio +rank0 [2026-11-14 06:50:38,725] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 06:50:38,725] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:51:39,715] INFO : Step 41000/500000 rank0 | Train Loss: 6.9763 | Train Loss Unweighted: 3.5328 | Train Loss Components: Classification: 0.0236 | Classification_binary: 3.2180 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1332 | Regression_eta: 0.0058 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2638 | Classification_binary: 3.2180 | Regression_cos_phi: 0.0972 | Regression_energy: 1.4191 | Regression_eta: 0.0465 | Regression_pt: 1.8481 | Regression_sin_phi: 0.0835 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5217s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0378 | max_loss: 0.0028 | max_backward: 0.0416 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.1369 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2875.1 | uss_MB: 1807.7 | child_rss_MB: 22499.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:51:42,724] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 06:51:42,724] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:52:46,683] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 06:52:46,683] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:53:33,882] INFO : Step 42000/500000 rank0 | Train Loss: 7.3093 | Train Loss Unweighted: 3.4774 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.1213 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1385 | Regression_eta: 0.0219 | Regression_pt: 0.1448 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3326 | Classification_binary: 3.1213 | Regression_cos_phi: 0.0760 | Regression_energy: 1.4753 | Regression_eta: 0.1759 | Regression_pt: 1.9730 | Regression_sin_phi: 0.1552 | LR: 1.07e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5197s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0389 | max_loss: 0.0032 | max_backward: 0.0434 | max_optimizer_step: 0.0679 | max_tensorboard_logging: 0.1217 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2875.8 | uss_MB: 1808.4 | child_rss_MB: 22502.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 24.3 +rank0 [2026-11-14 06:53:33,884] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:53:33,956] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:53:50,964] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 06:53:50,964] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:54:55,101] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 06:54:55,101] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:55:28,197] INFO : Step 43000/500000 rank0 | Train Loss: 7.8496 | Train Loss Unweighted: 3.6933 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.3133 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1618 | Regression_eta: 0.0057 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3340 | Classification_binary: 3.3133 | Regression_cos_phi: 0.1049 | Regression_energy: 1.7242 | Regression_eta: 0.0461 | Regression_pt: 2.2077 | Regression_sin_phi: 0.1194 | LR: 1.11e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4960s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0637 | max_tensorboard_logging: 0.1213 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2878.7 | uss_MB: 1811.3 | child_rss_MB: 22505.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:55:59,193] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 06:55:59,193] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:57:02,551] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 06:57:02,551] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:57:21,561] INFO : Step 44000/500000 rank0 | Train Loss: 9.3359 | Train Loss Unweighted: 4.0338 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.5511 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2164 | Regression_eta: 0.0085 | Regression_pt: 0.2173 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2955 | Classification_binary: 3.5511 | Regression_cos_phi: 0.0889 | Regression_energy: 2.3057 | Regression_eta: 0.0682 | Regression_pt: 2.9606 | Regression_sin_phi: 0.0660 | LR: 1.15e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4950s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1221 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2879.1 | uss_MB: 1811.7 | child_rss_MB: 22507.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:57:21,563] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:57:21,634] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:58:06,743] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 06:58:06,743] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:59:10,714] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 06:59:10,715] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:59:15,625] INFO : Step 45000/500000 rank0 | Train Loss: 7.4542 | Train Loss Unweighted: 3.6780 | Train Loss Components: Classification: 0.0311 | Classification_binary: 3.3315 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1439 | Regression_eta: 0.0082 | Regression_pt: 0.1455 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3470 | Classification_binary: 3.3315 | Regression_cos_phi: 0.0825 | Regression_energy: 1.5333 | Regression_eta: 0.0659 | Regression_pt: 1.9830 | Regression_sin_phi: 0.1109 | LR: 1.19e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4977s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0027 | max_backward: 0.0441 | max_optimizer_step: 0.0671 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2879.2 | uss_MB: 1811.8 | child_rss_MB: 22505.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:00:14,596] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 07:00:14,596] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:01:07,453] INFO : Step 46000/500000 rank0 | Train Loss: 7.2933 | Train Loss Unweighted: 3.5687 | Train Loss Components: Classification: 0.0266 | Classification_binary: 3.2281 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1432 | Regression_eta: 0.0062 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2974 | Classification_binary: 3.2281 | Regression_cos_phi: 0.1068 | Regression_energy: 1.5255 | Regression_eta: 0.0499 | Regression_pt: 1.9917 | Regression_sin_phi: 0.0939 | LR: 1.23e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5375s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0385 | max_loss: 0.0032 | max_backward: 0.0470 | max_optimizer_step: 0.0709 | max_tensorboard_logging: 0.1261 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2875.9 | uss_MB: 1808.5 | child_rss_MB: 22507.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:01:07,455] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:01:07,529] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 07:01:17,430] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 07:01:17,430] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:02:22,231] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 07:02:22,231] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:03:00,939] INFO : Step 47000/500000 rank0 | Train Loss: 7.8079 | Train Loss Unweighted: 3.7243 | Train Loss Components: Classification: 0.0346 | Classification_binary: 3.3497 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1526 | Regression_eta: 0.0079 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3866 | Classification_binary: 3.3497 | Regression_cos_phi: 0.1163 | Regression_energy: 1.6255 | Regression_eta: 0.0638 | Regression_pt: 2.1296 | Regression_sin_phi: 0.1365 | LR: 1.27e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4937s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0027 | max_backward: 0.0395 | max_optimizer_step: 0.0627 | max_tensorboard_logging: 0.1352 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2878.7 | uss_MB: 1811.3 | child_rss_MB: 22512.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:03:23,519] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 07:03:23,519] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:27,528] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 07:04:27,528] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:53,830] INFO : Step 48000/500000 rank0 | Train Loss: 7.0421 | Train Loss Unweighted: 3.5633 | Train Loss Components: Classification: 0.0320 | Classification_binary: 3.2445 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1342 | Regression_eta: 0.0064 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3570 | Classification_binary: 3.2445 | Regression_cos_phi: 0.0769 | Regression_energy: 1.4301 | Regression_eta: 0.0512 | Regression_pt: 1.8214 | Regression_sin_phi: 0.0610 | LR: 1.31e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5114s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0005 | max_forward: 0.0371 | max_loss: 0.0030 | max_backward: 0.0426 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1278 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2878.8 | uss_MB: 1811.4 | child_rss_MB: 22510.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:04:53,832] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:04:53,913] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 07:05:31,568] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 07:05:31,568] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:35,219] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 07:06:35,220] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:47,607] INFO : Step 49000/500000 rank0 | Train Loss: 7.1939 | Train Loss Unweighted: 3.5544 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.2200 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1411 | Regression_eta: 0.0113 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3028 | Classification_binary: 3.2200 | Regression_cos_phi: 0.0685 | Regression_energy: 1.5029 | Regression_eta: 0.0913 | Regression_pt: 1.9407 | Regression_sin_phi: 0.0676 | LR: 1.36e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5107s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0027 | max_backward: 0.0418 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2878.8 | uss_MB: 1811.4 | child_rss_MB: 22508.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:07:39,224] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 07:07:39,224] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:08:41,446] INFO : Step 50000/500000 rank0 | Train Loss: 7.9581 | Train Loss Unweighted: 3.5685 | Train Loss Components: Classification: 0.0316 | Classification_binary: 3.1659 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1726 | Regression_eta: 0.0088 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3524 | Classification_binary: 3.1659 | Regression_cos_phi: 0.1225 | Regression_energy: 1.8386 | Regression_eta: 0.0705 | Regression_pt: 2.3054 | Regression_sin_phi: 0.1029 | LR: 1.40e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5013s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0030 | max_backward: 0.0407 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2875.3 | uss_MB: 1807.9 | child_rss_MB: 22512.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:08:41,448] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:08:41,517] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 07:08:41,518] INFO : Running validation on rank0 +rank0 [2026-11-14 07:08:41,518] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:08:41,518] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:08:41,668] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 07:08:41,795] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:08:41,915] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:08:41,986] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:08:42,055] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:08:42,125] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:08:42,199] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:08:42,269] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:08:42,374] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:08:42,494] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:08:42,616] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:08:42,687] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:08:42,758] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:08:42,825] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:08:42,895] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:08:42,964] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:08:43,069] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:08:43,076] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:08:43,079] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:08:43,081] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:08:43,083] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:08:43,083] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:08:43,087] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:08:43,089] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:08:43,091] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:08:43,093] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:08:43,095] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:08:43,097] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:08:43,099] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:08:43,101] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:08:43,103] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:08:43,105] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:08:44,698] INFO : VALIDATION | Step=50000/500000 | Train Loss=7.9581 | Valid Loss=8.4101 | Train Loss Unweighted=3.5685 | Valid Loss Unweighted=3.7656 | Stale=0 | ETA=854.6m +rank0 [2026-11-14 07:08:44,698] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:08:45,074] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:08:45,109] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:45,158] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:08:45,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:08:45,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:08:46,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:08:46,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:08:46,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:08:46,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:08:47,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:08:47,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:08:47,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:08:48,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:08:48,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:08:48,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:08:48,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:08:49,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:08:49,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:08:49,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:08:49,605] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:08:49,637] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:50,254] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:50,649] INFO : Plotted number of elements +rank0 [2026-11-14 07:08:52,021] INFO : Plotted jets +rank0 [2026-11-14 07:08:53,037] INFO : Plotted jet ratio +rank0 [2026-11-14 07:08:55,562] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 07:08:55,562] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:09:58,946] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 07:09:58,946] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:10:46,790] INFO : Step 51000/500000 rank0 | Train Loss: 7.2487 | Train Loss Unweighted: 3.5729 | Train Loss Components: Classification: 0.0288 | Classification_binary: 3.2342 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1389 | Regression_eta: 0.0105 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3218 | Classification_binary: 3.2342 | Regression_cos_phi: 0.1083 | Regression_energy: 1.4799 | Regression_eta: 0.0847 | Regression_pt: 1.8928 | Regression_sin_phi: 0.1269 | LR: 1.44e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4946s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1192 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3007.4 | uss_MB: 1940.1 | child_rss_MB: 22515.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:11:02,852] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 07:11:02,852] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:12:06,013] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 07:12:06,014] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:12:39,691] INFO : Step 52000/500000 rank0 | Train Loss: 6.7873 | Train Loss Unweighted: 3.5016 | Train Loss Components: Classification: 0.0340 | Classification_binary: 3.1996 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1214 | Regression_eta: 0.0057 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3794 | Classification_binary: 3.1996 | Regression_cos_phi: 0.0943 | Regression_energy: 1.2940 | Regression_eta: 0.0459 | Regression_pt: 1.6624 | Regression_sin_phi: 0.1117 | LR: 1.49e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5084s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0030 | max_backward: 0.0420 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1226 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3007.5 | uss_MB: 1940.1 | child_rss_MB: 22516.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:12:39,693] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:12:39,758] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:13:09,719] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 07:13:09,719] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:14:13,175] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 07:14:13,175] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:14:32,974] INFO : Step 53000/500000 rank0 | Train Loss: 7.2691 | Train Loss Unweighted: 3.7245 | Train Loss Components: Classification: 0.0386 | Classification_binary: 3.3986 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1309 | Regression_eta: 0.0085 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4303 | Classification_binary: 3.3986 | Regression_cos_phi: 0.0932 | Regression_energy: 1.3951 | Regression_eta: 0.0685 | Regression_pt: 1.8158 | Regression_sin_phi: 0.0676 | LR: 1.53e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5154s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0031 | max_backward: 0.0411 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1249 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3008.2 | uss_MB: 1940.8 | child_rss_MB: 22516.4 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 74.3 +rank0 [2026-11-14 07:15:16,840] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 07:15:16,840] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:20,073] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 07:16:20,073] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:25,861] INFO : Step 54000/500000 rank0 | Train Loss: 6.3539 | Train Loss Unweighted: 3.3410 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.0633 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1131 | Regression_eta: 0.0095 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3256 | Classification_binary: 3.0633 | Regression_cos_phi: 0.0694 | Regression_energy: 1.2055 | Regression_eta: 0.0763 | Regression_pt: 1.5534 | Regression_sin_phi: 0.0603 | LR: 1.58e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4924s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0030 | max_backward: 0.0406 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1200 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2901.0 | uss_MB: 1833.6 | child_rss_MB: 22516.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:16:25,863] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:16:25,930] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:17:23,494] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 07:17:23,494] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:18:18,685] INFO : Step 55000/500000 rank0 | Train Loss: 6.6991 | Train Loss Unweighted: 3.5704 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.2818 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1181 | Regression_eta: 0.0085 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0124 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2694 | Classification_binary: 3.2818 | Regression_cos_phi: 0.0927 | Regression_energy: 1.2578 | Regression_eta: 0.0686 | Regression_pt: 1.5948 | Regression_sin_phi: 0.1340 | LR: 1.62e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4959s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0376 | max_loss: 0.0030 | max_backward: 0.0419 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1207 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2903.4 | uss_MB: 1836.0 | child_rss_MB: 22521.7 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:18:27,223] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 07:18:27,224] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:19:30,315] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 07:19:30,315] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:20:11,285] INFO : Step 56000/500000 rank0 | Train Loss: 6.9033 | Train Loss Unweighted: 3.4739 | Train Loss Components: Classification: 0.0312 | Classification_binary: 3.1586 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1323 | Regression_eta: 0.0068 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3477 | Classification_binary: 3.1586 | Regression_cos_phi: 0.1076 | Regression_energy: 1.4099 | Regression_eta: 0.0545 | Regression_pt: 1.7567 | Regression_sin_phi: 0.0683 | LR: 1.67e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5026s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0030 | max_backward: 0.0406 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2903.7 | uss_MB: 1836.3 | child_rss_MB: 22521.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:20:11,287] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:20:11,355] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:20:33,702] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 07:20:33,703] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:21:36,858] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 07:21:36,858] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:22:04,178] INFO : Step 57000/500000 rank0 | Train Loss: 5.9924 | Train Loss Unweighted: 3.1726 | Train Loss Components: Classification: 0.0306 | Classification_binary: 2.9128 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1074 | Regression_eta: 0.0063 | Regression_pt: 0.1043 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3419 | Classification_binary: 2.9128 | Regression_cos_phi: 0.0576 | Regression_energy: 1.1440 | Regression_eta: 0.0509 | Regression_pt: 1.4209 | Regression_sin_phi: 0.0643 | LR: 1.72e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5087s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0029 | max_backward: 0.0409 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1309 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2903.7 | uss_MB: 1836.3 | child_rss_MB: 22517.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:22:41,039] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 07:22:41,039] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:23:44,523] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 07:23:44,523] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:23:57,835] INFO : Step 58000/500000 rank0 | Train Loss: 5.8144 | Train Loss Unweighted: 3.1276 | Train Loss Components: Classification: 0.0309 | Classification_binary: 2.8801 | Regression_cos_phi: 0.0079 | Regression_energy: 0.0973 | Regression_eta: 0.0064 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3443 | Classification_binary: 2.8801 | Regression_cos_phi: 0.0867 | Regression_energy: 1.0366 | Regression_eta: 0.0512 | Regression_pt: 1.3504 | Regression_sin_phi: 0.0651 | LR: 1.76e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.4970s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0407 | max_loss: 0.0029 | max_backward: 0.0410 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1227 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2900.4 | uss_MB: 1833.0 | child_rss_MB: 22519.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:23:57,837] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:23:57,907] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:24:48,615] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 07:24:48,615] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:25:51,076] INFO : Step 59000/500000 rank0 | Train Loss: 7.3318 | Train Loss Unweighted: 3.3716 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.0056 | Regression_cos_phi: 0.0206 | Regression_energy: 0.1458 | Regression_eta: 0.0133 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3118 | Classification_binary: 3.0056 | Regression_cos_phi: 0.2276 | Regression_energy: 1.5534 | Regression_eta: 0.1070 | Regression_pt: 2.0076 | Regression_sin_phi: 0.1188 | LR: 1.81e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.4959s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0371 | max_loss: 0.0027 | max_backward: 0.0410 | max_optimizer_step: 0.0648 | max_tensorboard_logging: 0.1251 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2902.6 | uss_MB: 1835.2 | child_rss_MB: 22520.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:25:52,194] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 07:25:52,194] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:26:55,412] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 07:26:55,413] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:27:44,653] INFO : Step 60000/500000 rank0 | Train Loss: 6.2471 | Train Loss Unweighted: 3.3897 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.1248 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1079 | Regression_eta: 0.0123 | Regression_pt: 0.1064 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2481 | Classification_binary: 3.1248 | Regression_cos_phi: 0.0883 | Regression_energy: 1.1498 | Regression_eta: 0.0993 | Regression_pt: 1.4495 | Regression_sin_phi: 0.0873 | LR: 1.86e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.4936s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0374 | max_loss: 0.0032 | max_backward: 0.0421 | max_optimizer_step: 0.0661 | max_tensorboard_logging: 0.1188 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2902.8 | uss_MB: 1835.4 | child_rss_MB: 22517.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:27:44,655] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:27:44,725] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:27:44,726] INFO : Running validation on rank0 +rank0 [2026-11-14 07:27:44,726] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:27:44,726] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:27:44,875] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 07:27:45,001] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:27:45,122] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:27:45,193] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:27:45,260] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:27:45,326] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:27:45,395] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:27:45,465] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:27:45,582] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:27:45,702] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:27:45,820] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:27:45,889] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:27:45,956] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:27:46,023] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:27:46,092] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:27:46,161] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:27:46,264] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:27:46,271] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:27:46,273] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:27:46,276] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:27:46,278] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:27:46,280] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:27:46,282] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:27:46,284] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:27:46,286] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:27:46,288] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:27:46,290] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:27:46,292] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:27:46,294] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:27:46,296] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:27:46,297] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:27:46,300] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:27:47,822] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.2471 | Valid Loss=8.7496 | Train Loss Unweighted=3.3897 | Valid Loss Unweighted=3.7868 | Stale=1 | ETA=836.1m +rank0 [2026-11-14 07:27:47,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:27:48,161] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:27:48,170] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:48,215] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:27:48,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:27:48,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:27:49,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:27:49,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:27:49,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:27:50,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:27:50,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:27:50,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:27:50,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:27:51,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:27:51,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:27:51,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:27:51,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:27:52,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:27:52,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:27:52,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:27:52,655] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:27:52,661] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:53,251] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:53,633] INFO : Plotted number of elements +rank0 [2026-11-14 07:27:54,993] INFO : Plotted jets +rank0 [2026-11-14 07:27:55,986] INFO : Plotted jet ratio +rank0 [2026-11-14 07:28:11,530] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 07:28:11,530] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:13,135] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 07:29:13,135] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:48,893] INFO : Step 61000/500000 rank0 | Train Loss: 6.3731 | Train Loss Unweighted: 3.5179 | Train Loss Components: Classification: 0.0259 | Classification_binary: 3.2543 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1071 | Regression_eta: 0.0068 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2893 | Classification_binary: 3.2543 | Regression_cos_phi: 0.1254 | Regression_energy: 1.1413 | Regression_eta: 0.0551 | Regression_pt: 1.4167 | Regression_sin_phi: 0.0911 | LR: 1.91e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5294s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0034 | max_backward: 0.0431 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.1258 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3027.7 | uss_MB: 1960.3 | child_rss_MB: 22517.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 28.3 +rank0 [2026-11-14 07:30:19,624] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 07:30:19,624] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:31:21,144] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 07:31:21,144] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:31:41,226] INFO : Step 62000/500000 rank0 | Train Loss: 6.7947 | Train Loss Unweighted: 3.4449 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.1382 | Regression_cos_phi: 0.0044 | Regression_energy: 0.1276 | Regression_eta: 0.0083 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2932 | Classification_binary: 3.1382 | Regression_cos_phi: 0.0488 | Regression_energy: 1.3593 | Regression_eta: 0.0667 | Regression_pt: 1.8054 | Regression_sin_phi: 0.0832 | LR: 1.95e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5009s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0374 | max_loss: 0.0032 | max_backward: 0.0413 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1191 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2926.1 | uss_MB: 1858.7 | child_rss_MB: 22517.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:31:41,228] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:31:41,298] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:32:24,992] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 07:32:24,992] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:33:29,004] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 07:33:29,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:33:35,798] INFO : Step 63000/500000 rank0 | Train Loss: 6.4852 | Train Loss Unweighted: 3.3940 | Train Loss Components: Classification: 0.0194 | Classification_binary: 3.1096 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1172 | Regression_eta: 0.0084 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2166 | Classification_binary: 3.1096 | Regression_cos_phi: 0.1112 | Regression_energy: 1.2491 | Regression_eta: 0.0679 | Regression_pt: 1.6128 | Regression_sin_phi: 0.1180 | LR: 2.00e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5138s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0394 | max_loss: 0.0031 | max_backward: 0.0461 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.1236 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2929.6 | uss_MB: 1862.2 | child_rss_MB: 22524.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:34:33,337] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 07:34:33,337] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:35:30,283] INFO : Step 64000/500000 rank0 | Train Loss: 6.0350 | Train Loss Unweighted: 3.2915 | Train Loss Components: Classification: 0.0250 | Classification_binary: 3.0390 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1003 | Regression_eta: 0.0073 | Regression_pt: 0.1035 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2789 | Classification_binary: 3.0390 | Regression_cos_phi: 0.1172 | Regression_energy: 1.0687 | Regression_eta: 0.0588 | Regression_pt: 1.4109 | Regression_sin_phi: 0.0614 | LR: 2.05e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5254s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0402 | max_loss: 0.0031 | max_backward: 0.0407 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1365 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2929.7 | uss_MB: 1862.3 | child_rss_MB: 22519.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:35:30,285] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:35:30,358] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:35:38,112] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 07:35:38,112] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:36:42,256] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 07:36:42,257] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:37:25,033] INFO : Step 65000/500000 rank0 | Train Loss: 6.5617 | Train Loss Unweighted: 3.3415 | Train Loss Components: Classification: 0.0231 | Classification_binary: 3.0467 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1290 | Regression_eta: 0.0052 | Regression_pt: 0.1255 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2574 | Classification_binary: 3.0467 | Regression_cos_phi: 0.0590 | Regression_energy: 1.3744 | Regression_eta: 0.0419 | Regression_pt: 1.7105 | Regression_sin_phi: 0.0718 | LR: 2.10e-04 | DataLoad Time: 0.0118s | Model Forward Time: 0.5157s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0384 | max_loss: 0.0032 | max_backward: 0.0426 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2929.9 | uss_MB: 1862.5 | child_rss_MB: 22520.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:37:46,702] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 07:37:46,702] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:38:50,423] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 07:38:50,423] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:39:18,677] INFO : Step 66000/500000 rank0 | Train Loss: 6.4530 | Train Loss Unweighted: 3.2832 | Train Loss Components: Classification: 0.0290 | Classification_binary: 2.9930 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1181 | Regression_eta: 0.0044 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3237 | Classification_binary: 2.9930 | Regression_cos_phi: 0.1270 | Regression_energy: 1.2581 | Regression_eta: 0.0352 | Regression_pt: 1.6476 | Regression_sin_phi: 0.0684 | LR: 2.15e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5049s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0407 | max_loss: 0.0031 | max_backward: 0.0410 | max_optimizer_step: 0.0655 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2925.5 | uss_MB: 1858.1 | child_rss_MB: 22521.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:39:18,678] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:39:18,751] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:39:54,418] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 07:39:54,419] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:40:57,911] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 07:40:57,911] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:41:12,226] INFO : Step 67000/500000 rank0 | Train Loss: 5.7913 | Train Loss Unweighted: 3.2797 | Train Loss Components: Classification: 0.0245 | Classification_binary: 3.0458 | Regression_cos_phi: 0.0148 | Regression_energy: 0.0887 | Regression_eta: 0.0095 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2738 | Classification_binary: 3.0458 | Regression_cos_phi: 0.1635 | Regression_energy: 0.9453 | Regression_eta: 0.0765 | Regression_pt: 1.1883 | Regression_sin_phi: 0.0982 | LR: 2.20e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5182s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0384 | max_loss: 0.0028 | max_backward: 0.0417 | max_optimizer_step: 0.0635 | max_tensorboard_logging: 0.1375 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2929.1 | uss_MB: 1861.8 | child_rss_MB: 22528.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:42:02,452] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 07:42:02,452] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:43:06,550] INFO : Step 68000/500000 rank0 | Train Loss: 5.6031 | Train Loss Unweighted: 3.2827 | Train Loss Components: Classification: 0.0206 | Classification_binary: 3.0673 | Regression_cos_phi: 0.0137 | Regression_energy: 0.0836 | Regression_eta: 0.0075 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2299 | Classification_binary: 3.0673 | Regression_cos_phi: 0.1512 | Regression_energy: 0.8903 | Regression_eta: 0.0604 | Regression_pt: 1.1165 | Regression_sin_phi: 0.0874 | LR: 2.25e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5079s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0397 | max_loss: 0.0031 | max_backward: 0.0423 | max_optimizer_step: 0.0655 | max_tensorboard_logging: 0.1226 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2929.3 | uss_MB: 1861.9 | child_rss_MB: 22527.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:43:06,552] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:43:06,622] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:43:06,846] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 07:43:06,846] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:44:10,801] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 07:44:10,802] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:45:00,571] INFO : Step 69000/500000 rank0 | Train Loss: 5.4339 | Train Loss Unweighted: 2.9482 | Train Loss Components: Classification: 0.0281 | Classification_binary: 2.7169 | Regression_cos_phi: 0.0106 | Regression_energy: 0.0901 | Regression_eta: 0.0090 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3133 | Classification_binary: 2.7169 | Regression_cos_phi: 0.1170 | Regression_energy: 0.9595 | Regression_eta: 0.0727 | Regression_pt: 1.1776 | Regression_sin_phi: 0.0768 | LR: 2.30e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4937s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0031 | max_backward: 0.0412 | max_optimizer_step: 0.0647 | max_tensorboard_logging: 0.1254 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2929.4 | uss_MB: 1862.0 | child_rss_MB: 22528.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:45:14,781] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 07:45:14,782] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:46:18,409] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 07:46:18,409] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:46:54,544] INFO : Step 70000/500000 rank0 | Train Loss: 6.2062 | Train Loss Unweighted: 3.2431 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.9678 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1119 | Regression_eta: 0.0130 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2756 | Classification_binary: 2.9678 | Regression_cos_phi: 0.1006 | Regression_energy: 1.1927 | Regression_eta: 0.1046 | Regression_pt: 1.4770 | Regression_sin_phi: 0.0879 | LR: 2.35e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5101s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0029 | max_backward: 0.0429 | max_optimizer_step: 0.0689 | max_tensorboard_logging: 0.1238 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2925.5 | uss_MB: 1858.1 | child_rss_MB: 22531.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:46:54,546] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:46:54,615] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:46:54,616] INFO : Running validation on rank0 +rank0 [2026-11-14 07:46:54,616] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:46:54,616] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:46:54,765] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 07:46:54,887] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:46:55,003] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:46:55,075] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:46:55,142] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:46:55,209] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:46:55,279] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:46:55,349] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:46:55,448] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:46:55,568] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:46:55,686] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:46:55,756] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:46:55,825] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:46:55,893] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:46:55,961] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:46:56,029] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:46:56,135] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:46:56,142] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:46:56,145] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:46:56,147] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:46:56,149] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:46:56,151] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:46:56,153] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:46:56,155] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:46:56,157] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:46:56,159] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:46:56,161] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:46:56,163] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:46:56,166] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:46:56,168] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:46:56,170] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:46:56,172] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:46:57,688] INFO : VALIDATION | Step=70000/500000 | Train Loss=6.2062 | Valid Loss=9.2872 | Train Loss Unweighted=3.2431 | Valid Loss Unweighted=3.9482 | Stale=2 | ETA=818.1m +rank0 [2026-11-14 07:46:57,689] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:46:58,199] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:46:58,206] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:46:58,251] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:46:58,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:46:59,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:46:59,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:46:59,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:46:59,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:47:00,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:47:00,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:47:00,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:47:00,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:47:01,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:47:01,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:47:01,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:47:01,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:47:02,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:47:02,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:47:02,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:47:02,667] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:47:02,674] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:47:03,272] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:47:03,661] INFO : Plotted number of elements +rank0 [2026-11-14 07:47:05,017] INFO : Plotted jets +rank0 [2026-11-14 07:47:06,023] INFO : Plotted jet ratio +rank0 [2026-11-14 07:47:34,815] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 07:47:34,815] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:48:38,232] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 07:48:38,232] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:49:00,168] INFO : Step 71000/500000 rank0 | Train Loss: 5.2992 | Train Loss Unweighted: 2.9020 | Train Loss Components: Classification: 0.0248 | Classification_binary: 2.6804 | Regression_cos_phi: 0.0073 | Regression_energy: 0.0871 | Regression_eta: 0.0079 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2762 | Classification_binary: 2.6804 | Regression_cos_phi: 0.0806 | Regression_energy: 0.9282 | Regression_eta: 0.0637 | Regression_pt: 1.1982 | Regression_sin_phi: 0.0720 | LR: 2.40e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5202s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0028 | max_backward: 0.0400 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1377 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3058.0 | uss_MB: 1990.6 | child_rss_MB: 22530.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:49:42,426] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 07:49:42,427] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:50:45,479] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 07:50:45,479] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:50:53,134] INFO : Step 72000/500000 rank0 | Train Loss: 5.2080 | Train Loss Unweighted: 3.1557 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.9655 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0744 | Regression_eta: 0.0077 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2393 | Classification_binary: 2.9655 | Regression_cos_phi: 0.0662 | Regression_energy: 0.7930 | Regression_eta: 0.0616 | Regression_pt: 1.0178 | Regression_sin_phi: 0.0646 | LR: 2.45e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5100s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0030 | max_backward: 0.0422 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1213 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3058.0 | uss_MB: 1990.6 | child_rss_MB: 22536.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:50:53,136] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 07:50:53,206] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:51:49,403] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 07:51:49,403] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:52:46,634] INFO : Step 73000/500000 rank0 | Train Loss: 6.1049 | Train Loss Unweighted: 3.0808 | Train Loss Components: Classification: 0.0293 | Classification_binary: 2.7993 | Regression_cos_phi: 0.0228 | Regression_energy: 0.1027 | Regression_eta: 0.0119 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3269 | Classification_binary: 2.7993 | Regression_cos_phi: 0.2522 | Regression_energy: 1.0938 | Regression_eta: 0.0958 | Regression_pt: 1.4329 | Regression_sin_phi: 0.1041 | LR: 2.50e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5163s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0031 | max_backward: 0.0407 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1273 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3058.0 | uss_MB: 1990.6 | child_rss_MB: 22535.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:52:53,427] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 07:52:53,427] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:53:56,875] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 07:53:56,875] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:54:40,097] INFO : Step 74000/500000 rank0 | Train Loss: 5.6216 | Train Loss Unweighted: 3.1573 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.9315 | Regression_cos_phi: 0.0061 | Regression_energy: 0.0935 | Regression_eta: 0.0034 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2586 | Classification_binary: 2.9315 | Regression_cos_phi: 0.0675 | Regression_energy: 0.9959 | Regression_eta: 0.0278 | Regression_pt: 1.2736 | Regression_sin_phi: 0.0667 | LR: 2.55e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5174s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1287 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2950.7 | uss_MB: 1883.3 | child_rss_MB: 22535.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:54:40,099] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 07:54:40,172] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:55:00,975] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 07:55:00,975] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:56:04,289] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 07:56:04,289] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:56:33,908] INFO : Step 75000/500000 rank0 | Train Loss: 5.6161 | Train Loss Unweighted: 2.8479 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.5940 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1043 | Regression_eta: 0.0059 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2371 | Classification_binary: 2.5940 | Regression_cos_phi: 0.0895 | Regression_energy: 1.1109 | Regression_eta: 0.0471 | Regression_pt: 1.4530 | Regression_sin_phi: 0.0844 | LR: 2.60e-04 | DataLoad Time: 0.0055s | Model Forward Time: 0.5153s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0392 | max_loss: 0.0030 | max_backward: 0.0416 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1304 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2953.2 | uss_MB: 1885.9 | child_rss_MB: 22533.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:57:07,870] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 07:57:07,870] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:58:11,414] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 07:58:11,414] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:58:27,159] INFO : Step 76000/500000 rank0 | Train Loss: 4.8464 | Train Loss Unweighted: 2.6441 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.4399 | Regression_cos_phi: 0.0083 | Regression_energy: 0.0787 | Regression_eta: 0.0086 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2666 | Classification_binary: 2.4399 | Regression_cos_phi: 0.0914 | Regression_energy: 0.8388 | Regression_eta: 0.0691 | Regression_pt: 1.0897 | Regression_sin_phi: 0.0508 | LR: 2.65e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5566s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0005 | max_forward: 0.0392 | max_loss: 0.0037 | max_backward: 0.0427 | max_optimizer_step: 0.0719 | max_tensorboard_logging: 0.1284 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2953.6 | uss_MB: 1886.2 | child_rss_MB: 22538.8 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:58:27,161] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 07:58:27,231] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:59:15,431] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 07:59:15,432] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:17,527] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 08:00:17,527] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:18,733] INFO : Step 77000/500000 rank0 | Train Loss: 4.4292 | Train Loss Unweighted: 2.5863 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.4156 | Regression_cos_phi: 0.0085 | Regression_energy: 0.0628 | Regression_eta: 0.0053 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2474 | Classification_binary: 2.4156 | Regression_cos_phi: 0.0942 | Regression_energy: 0.6691 | Regression_eta: 0.0427 | Regression_pt: 0.8874 | Regression_sin_phi: 0.0728 | LR: 2.70e-04 | DataLoad Time: 0.0004s | Model Forward Time: 0.5820s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0411 | max_loss: 0.0043 | max_backward: 0.0438 | max_optimizer_step: 0.0732 | max_tensorboard_logging: 0.1443 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2953.5 | uss_MB: 1886.2 | child_rss_MB: 22538.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 08:01:22,247] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 08:01:22,247] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:12,890] INFO : Step 78000/500000 rank0 | Train Loss: 5.2639 | Train Loss Unweighted: 2.8163 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.5899 | Regression_cos_phi: 0.0113 | Regression_energy: 0.0900 | Regression_eta: 0.0065 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2556 | Classification_binary: 2.5899 | Regression_cos_phi: 0.1248 | Regression_energy: 0.9591 | Regression_eta: 0.0525 | Regression_pt: 1.2012 | Regression_sin_phi: 0.0807 | LR: 2.75e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5110s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1307 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2953.5 | uss_MB: 1886.1 | child_rss_MB: 22541.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:02:12,892] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 08:02:12,962] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 08:02:26,268] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 08:02:26,268] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:03:29,920] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 08:03:29,920] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:04:06,490] INFO : Step 79000/500000 rank0 | Train Loss: 4.9621 | Train Loss Unweighted: 2.8435 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.6480 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0823 | Regression_eta: 0.0060 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2201 | Classification_binary: 2.6480 | Regression_cos_phi: 0.0380 | Regression_energy: 0.8769 | Regression_eta: 0.0482 | Regression_pt: 1.0778 | Regression_sin_phi: 0.0531 | LR: 2.80e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0378 | max_loss: 0.0032 | max_backward: 0.0413 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1209 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2953.2 | uss_MB: 1885.9 | child_rss_MB: 22541.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 08:04:33,828] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 08:04:33,828] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:37,421] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 08:05:37,421] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:59,933] INFO : Step 80000/500000 rank0 | Train Loss: 4.6948 | Train Loss Unweighted: 2.6752 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.4868 | Regression_cos_phi: 0.0065 | Regression_energy: 0.0717 | Regression_eta: 0.0092 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2489 | Classification_binary: 2.4868 | Regression_cos_phi: 0.0718 | Regression_energy: 0.7637 | Regression_eta: 0.0742 | Regression_pt: 0.9615 | Regression_sin_phi: 0.0880 | LR: 2.85e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4901s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0395 | max_optimizer_step: 0.0642 | max_tensorboard_logging: 0.1237 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2953.3 | uss_MB: 1886.0 | child_rss_MB: 22542.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:05:59,935] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 08:06:00,003] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 08:06:00,004] INFO : Running validation on rank0 +rank0 [2026-11-14 08:06:00,004] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 08:06:00,004] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 08:06:00,158] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 08:06:00,282] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 08:06:00,403] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 08:06:00,474] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 08:06:00,544] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 08:06:00,612] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 08:06:00,681] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 08:06:00,749] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 08:06:00,852] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 08:06:00,972] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 08:06:01,091] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 08:06:01,159] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 08:06:01,227] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 08:06:01,294] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 08:06:01,361] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 08:06:01,429] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 08:06:01,537] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 08:06:01,544] INFO : plotted pt_target histogram +rank0 [2026-11-14 08:06:01,547] INFO : plotted pt_pred histogram +rank0 [2026-11-14 08:06:01,549] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 08:06:01,550] INFO : plotted eta_target histogram +rank0 [2026-11-14 08:06:01,553] INFO : plotted eta_pred histogram +rank0 [2026-11-14 08:06:01,555] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 08:06:01,557] INFO : plotted sphi_target histogram +rank0 [2026-11-14 08:06:01,559] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 08:06:01,562] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 08:06:01,564] INFO : plotted cphi_target histogram +rank0 [2026-11-14 08:06:01,566] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 08:06:01,568] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 08:06:01,570] INFO : plotted energy_target histogram +rank0 [2026-11-14 08:06:01,572] INFO : plotted energy_pred histogram +rank0 [2026-11-14 08:06:01,575] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 08:06:03,101] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.6948 | Valid Loss=10.1646 | Train Loss Unweighted=2.6752 | Valid Loss Unweighted=4.3723 | Stale=3 | ETA=799.4m +rank0 [2026-11-14 08:06:03,101] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 08:06:03,512] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 08:06:03,519] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 08:06:03,562] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 08:06:04,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 08:06:04,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 08:06:04,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 08:06:04,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 08:06:05,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 08:06:05,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 08:06:05,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 08:06:05,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 08:06:06,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 08:06:06,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 08:06:06,636] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 08:06:06,903] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 08:06:07,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 08:06:07,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 08:06:07,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 08:06:07,916] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 08:06:07,981] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 08:06:07,988] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:06:08,577] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:06:08,973] INFO : Plotted number of elements +rank0 [2026-11-14 08:06:10,346] INFO : Plotted jets +rank0 [2026-11-14 08:06:11,361] INFO : Plotted jet ratio +rank0 [2026-11-14 08:06:11,813] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 08:06:11,813] INFO : Training completed. Total time on device 0: 152.409min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..d566cc8ab0fef332be81e1a707e2adbf340797d2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/train.log.0 @@ -0,0 +1,1351 @@ +rank0 [2026-11-14 05:33:45,194] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 05:33:45,194] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 05:33:45,197] INFO : Instantiating model +rank0 [2026-11-14 05:33:45,219] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 05:33:45,219] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 05:33:45,221] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 05:33:45,221] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 05:33:45,221] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 05:33:45,221] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 05:33:45,221] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,224] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 05:33:45,224] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,227] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 05:33:45,227] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,230] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 05:33:45,230] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,232] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 05:33:45,233] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,235] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 05:33:45,235] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,238] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 05:33:45,238] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 05:33:45,239] INFO : Initializing output DNNs +rank0 [2026-11-14 05:33:45,239] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,242] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:33:45,249] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 05:33:45,250] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 05:33:45,250] INFO : jagged_attention enabled=True +rank0 [2026-11-14 05:33:45,250] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 05:33:45,250] INFO : task_queries enabled=True +rank0 [2026-11-14 05:33:45,250] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 05:33:45,250] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 05:33:45,250] INFO : classification_query parameters: 256 +rank0 [2026-11-14 05:33:45,250] INFO : regression_query parameters: 256 +rank0 [2026-11-14 05:33:45,250] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 05:33:45,250] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 05:33:45,250] INFO : output_mode=elementwise +rank0 [2026-11-14 05:33:45,250] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 05:33:45,250] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 05:33:45,250] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 05:33:45,250] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 05:33:45,250] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 05:33:45,250] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 05:33:45,250] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 05:33:45,251] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 05:33:45,252] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 05:33:45,252] INFO : MLPF __init__ done +rank0 [2026-11-14 05:33:45,252] INFO : Instantiated model +rank0 [2026-11-14 05:33:45,252] INFO : Moving model to device rank=0 +rank0 [2026-11-14 05:33:45,261] INFO : Moved model to device rank=0 +rank0 [2026-11-14 05:33:45,262] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 05:33:45,360] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 05:33:45,360] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 05:33:45,361] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 05:33:45,361] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 05:33:45,361] INFO : Total parameters: 3910930 +rank0 [2026-11-14 05:33:45,364] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 05:33:45,367] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812 +rank0 [2026-11-14 05:33:45,367] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812 +rank0 [2026-11-14 05:33:45,367] INFO : Getting dataloaders +rank0 [2026-11-14 05:33:45,367] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:33:46,582] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,620] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,691] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,726] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,775] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,823] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,858] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,895] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,937] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,970] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:33:46,970] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 05:33:46,970] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 05:33:46,970] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 05:33:46,970] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:33:46,978] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:46,986] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:46,993] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,001] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,010] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,018] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,025] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,033] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,040] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,049] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:33:47,050] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 05:33:47,050] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 05:33:47,095] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:33:47,246] INFO : Got dataloaders +rank0 [2026-11-14 05:33:47,246] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 05:33:47,246] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 05:33:47,247] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 05:33:47,263] INFO : Creating train_iterator +rank0 [2026-11-14 05:33:47,264] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:33:47,291] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:33:47,436] INFO : Created train_iterator +rank0 [2026-11-14 05:34:00,297] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.106752347946167, 'Classification': 0.6368530941009521, 'Regression_pt': 0.5215235805511474, 'Regression_eta': 0.8829098019003868, 'Regression_sin_phi': 0.6576527345180512, 'Regression_cos_phi': 0.643730517923832, 'Regression_energy': 0.6670048692822457}, weights={'Classification_binary': 1.0, 'Classification': 11.159171104431152, 'Regression_pt': 13.62690544128418, 'Regression_eta': 8.049239158630371, 'Regression_sin_phi': 10.806238174438477, 'Regression_cos_phi': 11.039949417114258, 'Regression_energy': 10.654723167419434} +rank0 [2026-11-14 05:34:53,284] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 05:34:53,284] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:35:43,301] INFO : Step 1000/500000 rank0 | Train Loss: 11.1433 | Train Loss Unweighted: 5.0567 | Train Loss Components: Classification: 0.0634 | Classification_binary: 4.4926 | Regression_cos_phi: 0.0196 | Regression_energy: 0.2126 | Regression_eta: 0.0255 | Regression_pt: 0.2233 | Regression_sin_phi: 0.0197 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.7076 | Classification_binary: 4.4926 | Regression_cos_phi: 0.2167 | Regression_energy: 2.2653 | Regression_eta: 0.2053 | Regression_pt: 3.0431 | Regression_sin_phi: 0.2127 | LR: 2.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4830s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0344 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0628 | max_tensorboard_logging: 0.1388 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.4 | uss_MB: 1411.4 | child_rss_MB: 22481.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:35:56,611] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 05:35:56,612] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:00,710] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 05:37:00,710] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:37,200] INFO : Step 2000/500000 rank0 | Train Loss: 11.5924 | Train Loss Unweighted: 4.8458 | Train Loss Components: Classification: 0.0449 | Classification_binary: 4.2249 | Regression_cos_phi: 0.0192 | Regression_energy: 0.2507 | Regression_eta: 0.0251 | Regression_pt: 0.2638 | Regression_sin_phi: 0.0174 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.5006 | Classification_binary: 4.2249 | Regression_cos_phi: 0.2119 | Regression_energy: 2.6715 | Regression_eta: 0.2018 | Regression_pt: 3.5942 | Regression_sin_phi: 0.1876 | LR: 2.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4633s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0342 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0592 | max_tensorboard_logging: 0.1267 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2572.2 | uss_MB: 1413.2 | child_rss_MB: 22483.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:37:37,202] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:38:04,350] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 05:38:04,350] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:06,585] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 05:39:06,586] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:28,573] INFO : Step 3000/500000 rank0 | Train Loss: 10.1246 | Train Loss Unweighted: 4.8538 | Train Loss Components: Classification: 0.0435 | Classification_binary: 4.3694 | Regression_cos_phi: 0.0160 | Regression_energy: 0.1972 | Regression_eta: 0.0128 | Regression_pt: 0.2010 | Regression_sin_phi: 0.0139 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4855 | Classification_binary: 4.3694 | Regression_cos_phi: 0.1766 | Regression_energy: 2.1014 | Regression_eta: 0.1032 | Regression_pt: 2.7384 | Regression_sin_phi: 0.1500 | LR: 2.05e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5311s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0438 | max_loss: 0.0031 | max_backward: 0.0410 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1454 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2575.9 | uss_MB: 1417.4 | child_rss_MB: 22482.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:40:10,379] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 05:40:10,379] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:13,644] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 05:41:13,644] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:21,476] INFO : Step 4000/500000 rank0 | Train Loss: 10.5804 | Train Loss Unweighted: 4.8436 | Train Loss Components: Classification: 0.0411 | Classification_binary: 4.3151 | Regression_cos_phi: 0.0191 | Regression_energy: 0.2207 | Regression_eta: 0.0171 | Regression_pt: 0.2183 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4586 | Classification_binary: 4.3151 | Regression_cos_phi: 0.2105 | Regression_energy: 2.3515 | Regression_eta: 0.1380 | Regression_pt: 2.9751 | Regression_sin_phi: 0.1317 | LR: 2.08e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.5266s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0365 | max_loss: 0.0030 | max_backward: 0.0426 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1491 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2576.2 | uss_MB: 1417.7 | child_rss_MB: 22485.2 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 05:41:21,478] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:42:17,292] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 05:42:17,292] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:43:14,483] INFO : Step 5000/500000 rank0 | Train Loss: 9.9829 | Train Loss Unweighted: 4.4750 | Train Loss Components: Classification: 0.0471 | Classification_binary: 3.9661 | Regression_cos_phi: 0.0110 | Regression_energy: 0.2068 | Regression_eta: 0.0227 | Regression_pt: 0.2102 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.5253 | Classification_binary: 3.9661 | Regression_cos_phi: 0.1211 | Regression_energy: 2.2037 | Regression_eta: 0.1827 | Regression_pt: 2.8649 | Regression_sin_phi: 0.1191 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4882s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0002 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0592 | max_tensorboard_logging: 0.1393 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2575.9 | uss_MB: 1417.4 | child_rss_MB: 22485.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:43:20,979] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 05:43:20,979] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:24,058] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 05:44:24,058] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:45:07,054] INFO : Step 6000/500000 rank0 | Train Loss: 9.5371 | Train Loss Unweighted: 4.4290 | Train Loss Components: Classification: 0.0303 | Classification_binary: 3.9625 | Regression_cos_phi: 0.0077 | Regression_energy: 0.2016 | Regression_eta: 0.0100 | Regression_pt: 0.2056 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3378 | Classification_binary: 3.9625 | Regression_cos_phi: 0.0854 | Regression_energy: 2.1478 | Regression_eta: 0.0809 | Regression_pt: 2.8014 | Regression_sin_phi: 0.1214 | LR: 2.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4766s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0357 | max_loss: 0.0027 | max_backward: 0.0391 | max_optimizer_step: 0.0616 | max_tensorboard_logging: 0.1184 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2576.2 | uss_MB: 1417.7 | child_rss_MB: 22486.4 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 7.9 +rank0 [2026-11-14 05:45:07,056] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:45:27,674] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 05:45:27,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:30,754] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 05:46:30,755] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:59,976] INFO : Step 7000/500000 rank0 | Train Loss: 9.5948 | Train Loss Unweighted: 4.4012 | Train Loss Components: Classification: 0.0388 | Classification_binary: 3.9239 | Regression_cos_phi: 0.0165 | Regression_energy: 0.1925 | Regression_eta: 0.0129 | Regression_pt: 0.1988 | Regression_sin_phi: 0.0178 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4328 | Classification_binary: 3.9239 | Regression_cos_phi: 0.1820 | Regression_energy: 2.0506 | Regression_eta: 0.1042 | Regression_pt: 2.7093 | Regression_sin_phi: 0.1919 | LR: 2.26e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5081s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0372 | max_loss: 0.0029 | max_backward: 0.0419 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1233 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2576.0 | uss_MB: 1417.5 | child_rss_MB: 22487.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:47:34,934] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 05:47:34,935] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:38,181] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 05:48:38,181] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:53,445] INFO : Step 8000/500000 rank0 | Train Loss: 9.4272 | Train Loss Unweighted: 4.3602 | Train Loss Components: Classification: 0.0430 | Classification_binary: 3.8964 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1993 | Regression_eta: 0.0074 | Regression_pt: 0.1959 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4802 | Classification_binary: 3.8964 | Regression_cos_phi: 0.0956 | Regression_energy: 2.1238 | Regression_eta: 0.0595 | Regression_pt: 2.6692 | Regression_sin_phi: 0.1024 | LR: 2.34e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5173s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0370 | max_loss: 0.0029 | max_backward: 0.0413 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1346 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2576.1 | uss_MB: 1417.6 | child_rss_MB: 22487.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 28.3 +rank0 [2026-11-14 05:48:53,447] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:48:53,518] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:49:41,783] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 05:49:41,784] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:45,084] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 05:50:45,084] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:50:46,390] INFO : Step 9000/500000 rank0 | Train Loss: 9.7958 | Train Loss Unweighted: 4.5659 | Train Loss Components: Classification: 0.0366 | Classification_binary: 4.0830 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2002 | Regression_eta: 0.0194 | Regression_pt: 0.1997 | Regression_sin_phi: 0.0164 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4087 | Classification_binary: 4.0830 | Regression_cos_phi: 0.1168 | Regression_energy: 2.1332 | Regression_eta: 0.1563 | Regression_pt: 2.7207 | Regression_sin_phi: 0.1770 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.4892s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0029 | max_backward: 0.0407 | max_optimizer_step: 0.0645 | max_tensorboard_logging: 0.1206 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2573.9 | uss_MB: 1415.3 | child_rss_MB: 22485.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 05:51:48,820] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 05:51:48,821] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:39,797] INFO : Step 10000/500000 rank0 | Train Loss: 9.9879 | Train Loss Unweighted: 4.3750 | Train Loss Components: Classification: 0.0329 | Classification_binary: 3.8631 | Regression_cos_phi: 0.0108 | Regression_energy: 0.2252 | Regression_eta: 0.0081 | Regression_pt: 0.2253 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3669 | Classification_binary: 3.8631 | Regression_cos_phi: 0.1191 | Regression_energy: 2.3995 | Regression_eta: 0.0654 | Regression_pt: 3.0706 | Regression_sin_phi: 0.1032 | LR: 2.52e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.5194s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0385 | max_loss: 0.0032 | max_backward: 0.0411 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2574.4 | uss_MB: 1415.8 | child_rss_MB: 22488.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:52:39,799] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:52:39,869] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:52:39,870] INFO : Running validation on rank0 +rank0 [2026-11-14 05:52:39,870] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:52:39,870] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:52:40,088] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 05:52:40,284] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:52:40,368] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:52:40,425] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:52:40,482] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:52:40,529] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:52:40,583] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:52:40,638] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:52:40,688] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:52:40,766] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:52:40,846] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:52:40,901] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:52:40,960] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:52:41,010] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:52:41,068] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:52:41,124] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:52:41,179] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:52:41,184] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:52:41,185] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:52:41,188] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:52:41,190] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:52:41,191] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:52:41,195] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:52:41,197] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:52:41,198] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:52:41,201] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:52:41,201] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:52:41,203] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:52:41,207] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:52:41,209] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:52:41,211] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:52:41,213] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:52:42,806] INFO : VALIDATION | Step=10000/500000 | Train Loss=9.9879 | Valid Loss=9.7571 | Train Loss Unweighted=4.3750 | Valid Loss Unweighted=4.3510 | Stale=0 | ETA=927.4m +rank0 [2026-11-14 05:52:42,807] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:52:43,333] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:52:43,341] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:43,391] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:52:44,102] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:52:44,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:52:44,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:52:44,891] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:52:45,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:52:45,411] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:52:45,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:52:45,934] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:52:46,194] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:52:46,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:52:46,709] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:52:46,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:52:47,231] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:52:47,493] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:52:47,754] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:52:47,986] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:52:48,039] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 05:52:48,046] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:48,647] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:49,408] INFO : Plotted number of elements +rank0 [2026-11-14 05:52:50,846] INFO : Plotted jets +rank0 [2026-11-14 05:52:51,850] INFO : Plotted jet ratio +rank0 [2026-11-14 05:53:05,480] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 05:53:05,480] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:09,474] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 05:54:09,474] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:46,409] INFO : Step 11000/500000 rank0 | Train Loss: 9.1829 | Train Loss Unweighted: 4.2780 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.8318 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1946 | Regression_eta: 0.0074 | Regression_pt: 0.2013 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3331 | Classification_binary: 3.8318 | Regression_cos_phi: 0.0593 | Regression_energy: 2.0735 | Regression_eta: 0.0597 | Regression_pt: 2.7427 | Regression_sin_phi: 0.0828 | LR: 2.63e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5106s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0633 | max_tensorboard_logging: 0.1278 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2749.4 | uss_MB: 1605.7 | child_rss_MB: 22490.5 | allocated_MB: 125.7 | reserved_MB: 620.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:55:13,517] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 05:55:13,517] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:16,956] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 05:56:16,956] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:39,628] INFO : Step 12000/500000 rank0 | Train Loss: 7.7118 | Train Loss Unweighted: 3.7300 | Train Loss Components: Classification: 0.0281 | Classification_binary: 3.3671 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1562 | Regression_eta: 0.0053 | Regression_pt: 0.1594 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3138 | Classification_binary: 3.3671 | Regression_cos_phi: 0.0678 | Regression_energy: 1.6641 | Regression_eta: 0.0423 | Regression_pt: 2.1723 | Regression_sin_phi: 0.0844 | LR: 2.75e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5104s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0031 | max_backward: 0.0401 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1245 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2749.6 | uss_MB: 1605.9 | child_rss_MB: 22490.6 | allocated_MB: 126.1 | reserved_MB: 640.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:56:39,630] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:56:39,704] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:57:20,636] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 05:57:20,636] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:24,102] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 05:58:24,102] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:32,890] INFO : Step 13000/500000 rank0 | Train Loss: 9.2807 | Train Loss Unweighted: 4.3454 | Train Loss Components: Classification: 0.0326 | Classification_binary: 3.8936 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1948 | Regression_eta: 0.0090 | Regression_pt: 0.1935 | Regression_sin_phi: 0.0132 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3635 | Classification_binary: 3.8936 | Regression_cos_phi: 0.0965 | Regression_energy: 2.0751 | Regression_eta: 0.0721 | Regression_pt: 2.6371 | Regression_sin_phi: 0.1428 | LR: 2.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5251s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0384 | max_loss: 0.0031 | max_backward: 0.0420 | max_optimizer_step: 0.0687 | max_tensorboard_logging: 0.1273 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2676.7 | uss_MB: 1533.0 | child_rss_MB: 22489.8 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:59:27,674] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:59:27,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:25,938] INFO : Step 14000/500000 rank0 | Train Loss: 9.3603 | Train Loss Unweighted: 4.2548 | Train Loss Components: Classification: 0.0337 | Classification_binary: 3.7887 | Regression_cos_phi: 0.0074 | Regression_energy: 0.1869 | Regression_eta: 0.0162 | Regression_pt: 0.2107 | Regression_sin_phi: 0.0112 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3766 | Classification_binary: 3.7887 | Regression_cos_phi: 0.0815 | Regression_energy: 1.9909 | Regression_eta: 0.1305 | Regression_pt: 2.8710 | Regression_sin_phi: 0.1211 | LR: 3.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5271s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0028 | max_backward: 0.0420 | max_optimizer_step: 0.0860 | max_tensorboard_logging: 0.1227 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2676.9 | uss_MB: 1533.2 | child_rss_MB: 22489.5 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:00:25,940] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:00:26,012] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 06:00:31,644] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 06:00:31,644] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:34,883] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 06:01:34,883] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:19,385] INFO : Step 15000/500000 rank0 | Train Loss: 9.0935 | Train Loss Unweighted: 4.2235 | Train Loss Components: Classification: 0.0324 | Classification_binary: 3.7785 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1883 | Regression_eta: 0.0111 | Regression_pt: 0.1956 | Regression_sin_phi: 0.0086 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3610 | Classification_binary: 3.7785 | Regression_cos_phi: 0.1006 | Regression_energy: 2.0061 | Regression_eta: 0.0892 | Regression_pt: 2.6649 | Regression_sin_phi: 0.0932 | LR: 3.17e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5162s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0375 | max_loss: 0.0032 | max_backward: 0.0420 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2679.9 | uss_MB: 1536.2 | child_rss_MB: 22491.6 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:02:38,974] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 06:02:38,974] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:42,612] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 06:03:42,612] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:11,862] INFO : Step 16000/500000 rank0 | Train Loss: 9.2886 | Train Loss Unweighted: 4.2352 | Train Loss Components: Classification: 0.0360 | Classification_binary: 3.7714 | Regression_cos_phi: 0.0138 | Regression_energy: 0.1938 | Regression_eta: 0.0131 | Regression_pt: 0.1966 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4019 | Classification_binary: 3.7714 | Regression_cos_phi: 0.1527 | Regression_energy: 2.0647 | Regression_eta: 0.1050 | Regression_pt: 2.6795 | Regression_sin_phi: 0.1134 | LR: 3.33e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4922s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0350 | max_loss: 0.0027 | max_backward: 0.0390 | max_optimizer_step: 0.0624 | max_tensorboard_logging: 0.1275 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2680.3 | uss_MB: 1536.6 | child_rss_MB: 22490.8 | allocated_MB: 126.1 | reserved_MB: 640.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 06:04:11,866] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:04:11,936] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 06:04:45,452] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 06:04:45,452] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:50,591] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 06:05:50,591] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:06:06,257] INFO : Step 17000/500000 rank0 | Train Loss: 9.3975 | Train Loss Unweighted: 4.2566 | Train Loss Components: Classification: 0.0354 | Classification_binary: 3.7848 | Regression_cos_phi: 0.0108 | Regression_energy: 0.1962 | Regression_eta: 0.0172 | Regression_pt: 0.2043 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3945 | Classification_binary: 3.7848 | Regression_cos_phi: 0.1189 | Regression_energy: 2.0903 | Regression_eta: 0.1385 | Regression_pt: 2.7840 | Regression_sin_phi: 0.0865 | LR: 3.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4665s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0002 | max_forward: 0.0340 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0624 | max_tensorboard_logging: 0.1217 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2676.1 | uss_MB: 1532.5 | child_rss_MB: 22493.2 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:52,400] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 06:06:52,401] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:56,367] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 06:07:56,367] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:58,588] INFO : Step 18000/500000 rank0 | Train Loss: 9.0548 | Train Loss Unweighted: 4.1751 | Train Loss Components: Classification: 0.0318 | Classification_binary: 3.7275 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1821 | Regression_eta: 0.0181 | Regression_pt: 0.1968 | Regression_sin_phi: 0.0099 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3549 | Classification_binary: 3.7275 | Regression_cos_phi: 0.0976 | Regression_energy: 1.9406 | Regression_eta: 0.1456 | Regression_pt: 2.6813 | Regression_sin_phi: 0.1074 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.5173s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0031 | max_backward: 0.0449 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1339 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2677.0 | uss_MB: 1533.3 | child_rss_MB: 22491.3 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:07:58,591] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:07:58,661] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 06:09:00,429] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 06:09:00,430] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:52,035] INFO : Step 19000/500000 rank0 | Train Loss: 8.4928 | Train Loss Unweighted: 4.1407 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.7439 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1672 | Regression_eta: 0.0061 | Regression_pt: 0.1739 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3347 | Classification_binary: 3.7439 | Regression_cos_phi: 0.1183 | Regression_energy: 1.7813 | Regression_eta: 0.0490 | Regression_pt: 2.3701 | Regression_sin_phi: 0.0955 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5150s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0030 | max_backward: 0.0413 | max_optimizer_step: 0.0751 | max_tensorboard_logging: 0.1210 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2679.9 | uss_MB: 1536.2 | child_rss_MB: 22492.1 | allocated_MB: 125.7 | reserved_MB: 642.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:10:04,267] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 06:10:04,267] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:07,551] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 06:11:07,551] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:47,234] INFO : Step 20000/500000 rank0 | Train Loss: 9.9339 | Train Loss Unweighted: 4.5901 | Train Loss Components: Classification: 0.0321 | Classification_binary: 4.1021 | Regression_cos_phi: 0.0103 | Regression_energy: 0.2105 | Regression_eta: 0.0102 | Regression_pt: 0.2145 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3581 | Classification_binary: 4.1021 | Regression_cos_phi: 0.1141 | Regression_energy: 2.2423 | Regression_eta: 0.0821 | Regression_pt: 2.9224 | Regression_sin_phi: 0.1129 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5597s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0032 | max_backward: 0.0417 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.1712 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2686.8 | uss_MB: 1543.2 | child_rss_MB: 22491.2 | allocated_MB: 125.7 | reserved_MB: 642.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:11:47,268] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:11:47,385] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:11:47,402] INFO : Running validation on rank0 +rank0 [2026-11-14 06:11:47,402] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:11:47,402] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:11:47,583] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 06:11:47,709] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:11:47,857] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:11:47,929] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:11:47,999] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:11:48,067] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:11:48,137] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:11:48,204] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:11:48,318] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:11:48,436] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:11:48,553] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:11:48,621] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:11:48,687] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:11:48,753] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:11:48,820] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:11:48,887] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:11:49,003] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:11:49,010] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:11:49,012] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:11:49,015] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:11:49,017] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:11:49,019] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:11:49,021] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:11:49,023] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:11:49,025] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:11:49,027] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:11:49,029] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:11:49,030] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:11:49,034] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:11:49,036] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:11:49,038] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:11:49,040] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:11:50,653] INFO : VALIDATION | Step=20000/500000 | Train Loss=9.9339 | Valid Loss=9.4105 | Train Loss Unweighted=4.5901 | Valid Loss Unweighted=4.1819 | Stale=0 | ETA=913.4m +rank0 [2026-11-14 06:11:50,654] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:11:51,086] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:11:51,117] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:51,217] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:11:51,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:11:51,944] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:11:52,218] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:11:52,475] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:11:52,728] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:11:52,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:11:53,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:11:53,496] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:11:53,744] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:11:54,000] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:11:54,259] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:11:54,518] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:11:54,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:11:55,029] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:11:55,296] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:11:55,523] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:11:55,573] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:11:55,579] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:56,165] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:56,564] INFO : Plotted number of elements +rank0 [2026-11-14 06:11:57,923] INFO : Plotted jets +rank0 [2026-11-14 06:11:58,930] INFO : Plotted jet ratio +rank0 [2026-11-14 06:12:25,574] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 06:12:25,575] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:28,586] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 06:13:28,586] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:51,988] INFO : Step 21000/500000 rank0 | Train Loss: 8.8431 | Train Loss Unweighted: 4.0429 | Train Loss Components: Classification: 0.0293 | Classification_binary: 3.6059 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1924 | Regression_eta: 0.0073 | Regression_pt: 0.1960 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3275 | Classification_binary: 3.6059 | Regression_cos_phi: 0.0670 | Regression_energy: 2.0504 | Regression_eta: 0.0584 | Regression_pt: 2.6706 | Regression_sin_phi: 0.0632 | LR: 4.28e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5283s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0371 | max_loss: 0.0032 | max_backward: 0.0404 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1384 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2722.1 | uss_MB: 1578.5 | child_rss_MB: 22493.1 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:14:31,580] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 06:14:31,580] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:34,294] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 06:15:34,295] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:43,914] INFO : Step 22000/500000 rank0 | Train Loss: 7.6051 | Train Loss Unweighted: 3.9735 | Train Loss Components: Classification: 0.0332 | Classification_binary: 3.6386 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1375 | Regression_eta: 0.0109 | Regression_pt: 0.1365 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3703 | Classification_binary: 3.6386 | Regression_cos_phi: 0.0937 | Regression_energy: 1.4649 | Regression_eta: 0.0881 | Regression_pt: 1.8596 | Regression_sin_phi: 0.0900 | LR: 4.50e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5138s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0413 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1289 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2734.3 | uss_MB: 1590.6 | child_rss_MB: 22493.7 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:15:43,916] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:15:43,989] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:16:37,976] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 06:16:37,976] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:36,210] INFO : Step 23000/500000 rank0 | Train Loss: 8.4608 | Train Loss Unweighted: 4.1721 | Train Loss Components: Classification: 0.0333 | Classification_binary: 3.7811 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1685 | Regression_eta: 0.0060 | Regression_pt: 0.1712 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3715 | Classification_binary: 3.7811 | Regression_cos_phi: 0.0682 | Regression_energy: 1.7955 | Regression_eta: 0.0486 | Regression_pt: 2.3325 | Regression_sin_phi: 0.0634 | LR: 4.73e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4880s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0031 | max_backward: 0.0400 | max_optimizer_step: 0.0660 | max_tensorboard_logging: 0.1162 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2738.2 | uss_MB: 1619.9 | child_rss_MB: 22494.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 38.3 +rank0 [2026-11-14 06:17:40,897] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 06:17:40,897] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:43,223] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 06:18:43,223] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:19:27,541] INFO : Step 24000/500000 rank0 | Train Loss: 8.4071 | Train Loss Unweighted: 3.9444 | Train Loss Components: Classification: 0.0358 | Classification_binary: 3.5361 | Regression_cos_phi: 0.0148 | Regression_energy: 0.1705 | Regression_eta: 0.0069 | Regression_pt: 0.1730 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3992 | Classification_binary: 3.5361 | Regression_cos_phi: 0.1631 | Regression_energy: 1.8167 | Regression_eta: 0.0557 | Regression_pt: 2.3568 | Regression_sin_phi: 0.0794 | LR: 4.97e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4828s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0027 | max_backward: 0.0394 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.1199 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2741.0 | uss_MB: 1646.7 | child_rss_MB: 22495.4 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 14.3 +rank0 [2026-11-14 06:19:27,543] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:19:27,616] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:19:46,064] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 06:19:46,064] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:48,693] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 06:20:48,693] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:21:19,615] INFO : Step 25000/500000 rank0 | Train Loss: 8.1063 | Train Loss Unweighted: 4.1059 | Train Loss Components: Classification: 0.0367 | Classification_binary: 3.7373 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1516 | Regression_eta: 0.0120 | Regression_pt: 0.1514 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4098 | Classification_binary: 3.7373 | Regression_cos_phi: 0.0951 | Regression_energy: 1.6153 | Regression_eta: 0.0963 | Regression_pt: 2.0635 | Regression_sin_phi: 0.0890 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4967s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0366 | max_loss: 0.0030 | max_backward: 0.0401 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1218 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2737.2 | uss_MB: 1651.6 | child_rss_MB: 22492.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:21:51,782] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 06:21:51,782] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:54,120] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 06:22:54,121] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:11,151] INFO : Step 26000/500000 rank0 | Train Loss: 8.5651 | Train Loss Unweighted: 3.9457 | Train Loss Components: Classification: 0.0366 | Classification_binary: 3.5231 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1812 | Regression_eta: 0.0080 | Regression_pt: 0.1807 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4090 | Classification_binary: 3.5231 | Regression_cos_phi: 0.0928 | Regression_energy: 1.9308 | Regression_eta: 0.0647 | Regression_pt: 2.4623 | Regression_sin_phi: 0.0825 | LR: 5.47e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5038s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0402 | max_optimizer_step: 0.0654 | max_tensorboard_logging: 0.1339 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2737.9 | uss_MB: 1652.3 | child_rss_MB: 22495.8 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:23:11,153] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:23:11,248] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:23:57,559] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 06:23:57,559] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:00,155] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 06:25:00,155] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:03,199] INFO : Step 27000/500000 rank0 | Train Loss: 10.6380 | Train Loss Unweighted: 4.4254 | Train Loss Components: Classification: 0.0389 | Classification_binary: 3.8600 | Regression_cos_phi: 0.0075 | Regression_energy: 0.2456 | Regression_eta: 0.0095 | Regression_pt: 0.2539 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4342 | Classification_binary: 3.8600 | Regression_cos_phi: 0.0826 | Regression_energy: 2.6164 | Regression_eta: 0.0767 | Regression_pt: 3.4593 | Regression_sin_phi: 0.1089 | LR: 5.74e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4811s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0394 | max_optimizer_step: 0.0635 | max_tensorboard_logging: 0.1207 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2740.9 | uss_MB: 1655.3 | child_rss_MB: 22495.0 | allocated_MB: 125.7 | reserved_MB: 634.0 | inactive_split_MB: 14.3 +rank0 [2026-11-14 06:26:03,298] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 06:26:03,298] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:54,965] INFO : Step 28000/500000 rank0 | Train Loss: 9.0425 | Train Loss Unweighted: 4.1518 | Train Loss Components: Classification: 0.0347 | Classification_binary: 3.7036 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1881 | Regression_eta: 0.0126 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0090 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3876 | Classification_binary: 3.7036 | Regression_cos_phi: 0.1212 | Regression_energy: 2.0043 | Regression_eta: 0.1012 | Regression_pt: 2.6271 | Regression_sin_phi: 0.0973 | LR: 6.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5167s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1233 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2741.2 | uss_MB: 1655.6 | child_rss_MB: 22495.1 | allocated_MB: 125.7 | reserved_MB: 634.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:26:54,967] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:26:55,045] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:27:06,174] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 06:27:06,174] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:08,513] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 06:28:08,513] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:46,205] INFO : Step 29000/500000 rank0 | Train Loss: 8.8674 | Train Loss Unweighted: 4.0665 | Train Loss Components: Classification: 0.0313 | Classification_binary: 3.6289 | Regression_cos_phi: 0.0125 | Regression_energy: 0.1834 | Regression_eta: 0.0075 | Regression_pt: 0.1928 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3493 | Classification_binary: 3.6289 | Regression_cos_phi: 0.1384 | Regression_energy: 1.9541 | Regression_eta: 0.0607 | Regression_pt: 2.6268 | Regression_sin_phi: 0.1092 | LR: 6.29e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4878s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0031 | max_backward: 0.0399 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1208 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2741.3 | uss_MB: 1664.4 | child_rss_MB: 22494.6 | allocated_MB: 125.7 | reserved_MB: 634.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:29:10,869] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 06:29:10,869] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:13,020] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 06:30:13,020] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:37,032] INFO : Step 30000/500000 rank0 | Train Loss: 8.2845 | Train Loss Unweighted: 4.0901 | Train Loss Components: Classification: 0.0393 | Classification_binary: 3.7052 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1591 | Regression_eta: 0.0093 | Regression_pt: 0.1612 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4389 | Classification_binary: 3.7052 | Regression_cos_phi: 0.0775 | Regression_energy: 1.6950 | Regression_eta: 0.0752 | Regression_pt: 2.1971 | Regression_sin_phi: 0.0956 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4910s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0386 | max_loss: 0.0029 | max_backward: 0.0414 | max_optimizer_step: 0.0648 | max_tensorboard_logging: 0.1230 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2738.4 | uss_MB: 1665.6 | child_rss_MB: 22496.2 | allocated_MB: 126.1 | reserved_MB: 634.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:30:37,034] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:30:37,104] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:30:37,105] INFO : Running validation on rank0 +rank0 [2026-11-14 06:30:37,105] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:30:37,105] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:30:37,259] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 06:30:37,377] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:30:37,492] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:30:37,561] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:30:37,629] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:30:37,696] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:30:37,762] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:30:37,828] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:30:37,944] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:30:38,063] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:30:38,178] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:30:38,245] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:30:38,313] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:30:38,379] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:30:38,447] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:30:38,512] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:30:38,625] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:30:38,632] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:30:38,634] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:30:38,637] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:30:38,639] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:30:38,641] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:30:38,643] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:30:38,645] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:30:38,647] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:30:38,649] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:30:38,651] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:30:38,653] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:30:38,655] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:30:38,657] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:30:38,659] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:30:38,661] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:30:40,236] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.2845 | Valid Loss=8.9936 | Train Loss Unweighted=4.0901 | Valid Loss Unweighted=4.0514 | Stale=0 | ETA=891.2m +rank0 [2026-11-14 06:30:40,236] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:30:40,591] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:30:40,600] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:40,645] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:30:41,162] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:30:41,424] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:30:41,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:30:41,941] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:30:42,199] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:30:42,455] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:30:42,707] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:30:42,961] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:30:43,230] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:30:43,485] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:30:43,739] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:30:44,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:30:44,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:30:44,514] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:30:44,770] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:30:45,001] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:30:45,053] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:30:45,061] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:45,644] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:46,030] INFO : Plotted number of elements +rank0 [2026-11-14 06:30:47,429] INFO : Plotted jets +rank0 [2026-11-14 06:30:48,487] INFO : Plotted jet ratio +rank0 [2026-11-14 06:31:28,087] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 06:31:28,087] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:29,843] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 06:32:29,843] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:39,833] INFO : Step 31000/500000 rank0 | Train Loss: 8.7836 | Train Loss Unweighted: 3.9801 | Train Loss Components: Classification: 0.0326 | Classification_binary: 3.5394 | Regression_cos_phi: 0.0143 | Regression_energy: 0.1814 | Regression_eta: 0.0112 | Regression_pt: 0.1859 | Regression_sin_phi: 0.0154 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3642 | Classification_binary: 3.5394 | Regression_cos_phi: 0.1575 | Regression_energy: 1.9323 | Regression_eta: 0.0900 | Regression_pt: 2.5338 | Regression_sin_phi: 0.1663 | LR: 6.88e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.4570s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0370 | max_loss: 0.0031 | max_backward: 0.0394 | max_optimizer_step: 0.0596 | max_tensorboard_logging: 0.1216 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2897.6 | uss_MB: 1830.2 | child_rss_MB: 22491.6 | allocated_MB: 126.1 | reserved_MB: 628.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 06:33:34,951] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 06:33:34,952] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:34,417] INFO : Step 32000/500000 rank0 | Train Loss: 10.0047 | Train Loss Unweighted: 4.1107 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0057 | Regression_energy: 0.2440 | Regression_eta: 0.0092 | Regression_pt: 0.2445 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3105 | Classification_binary: 3.5747 | Regression_cos_phi: 0.0628 | Regression_energy: 2.5993 | Regression_eta: 0.0742 | Regression_pt: 3.3314 | Regression_sin_phi: 0.0518 | LR: 7.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4976s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0031 | max_backward: 0.0407 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1207 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2906.3 | uss_MB: 1838.8 | child_rss_MB: 22490.7 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:34:34,419] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:34:34,490] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:34:38,164] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 06:34:38,164] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:41,083] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 06:35:41,083] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:26,994] INFO : Step 33000/500000 rank0 | Train Loss: 9.2693 | Train Loss Unweighted: 3.7727 | Train Loss Components: Classification: 0.0347 | Classification_binary: 3.2731 | Regression_cos_phi: 0.0056 | Regression_energy: 0.2225 | Regression_eta: 0.0047 | Regression_pt: 0.2238 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3872 | Classification_binary: 3.2731 | Regression_cos_phi: 0.0614 | Regression_energy: 2.3712 | Regression_eta: 0.0380 | Regression_pt: 3.0493 | Regression_sin_phi: 0.0890 | LR: 7.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5011s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0366 | max_loss: 0.0028 | max_backward: 0.0406 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1320 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.6 | uss_MB: 1763.2 | child_rss_MB: 22491.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:36:44,797] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 06:36:44,797] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:48,216] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 06:37:48,216] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:20,003] INFO : Step 34000/500000 rank0 | Train Loss: 9.1484 | Train Loss Unweighted: 4.0477 | Train Loss Components: Classification: 0.0357 | Classification_binary: 3.5836 | Regression_cos_phi: 0.0124 | Regression_energy: 0.1882 | Regression_eta: 0.0091 | Regression_pt: 0.2083 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3986 | Classification_binary: 3.5836 | Regression_cos_phi: 0.1371 | Regression_energy: 2.0049 | Regression_eta: 0.0730 | Regression_pt: 2.8388 | Regression_sin_phi: 0.1125 | LR: 7.83e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4847s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0390 | max_loss: 0.0027 | max_backward: 0.0410 | max_optimizer_step: 0.0655 | max_tensorboard_logging: 0.1230 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2849.7 | uss_MB: 1782.2 | child_rss_MB: 22492.9 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:38:20,005] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:38:20,075] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:38:51,650] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 06:38:51,650] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:39:54,603] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 06:39:54,604] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:12,751] INFO : Step 35000/500000 rank0 | Train Loss: 8.5985 | Train Loss Unweighted: 4.0657 | Train Loss Components: Classification: 0.0345 | Classification_binary: 3.6470 | Regression_cos_phi: 0.0134 | Regression_energy: 0.1771 | Regression_eta: 0.0150 | Regression_pt: 0.1699 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3855 | Classification_binary: 3.6470 | Regression_cos_phi: 0.1476 | Regression_energy: 1.8873 | Regression_eta: 0.1206 | Regression_pt: 2.3149 | Regression_sin_phi: 0.0957 | LR: 8.16e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4963s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0409 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1246 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2853.9 | uss_MB: 1786.4 | child_rss_MB: 22493.1 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:40:58,085] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 06:40:58,085] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:01,108] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 06:42:01,109] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:05,088] INFO : Step 36000/500000 rank0 | Train Loss: 6.5200 | Train Loss Unweighted: 3.3483 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.0579 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1220 | Regression_eta: 0.0058 | Regression_pt: 0.1232 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3143 | Classification_binary: 3.0579 | Regression_cos_phi: 0.0499 | Regression_energy: 1.2998 | Regression_eta: 0.0469 | Regression_pt: 1.6782 | Regression_sin_phi: 0.0729 | LR: 8.50e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5009s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0033 | max_backward: 0.0411 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1224 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2853.9 | uss_MB: 1786.5 | child_rss_MB: 22491.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:42:05,090] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:42:05,159] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:43:04,337] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 06:43:04,337] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:43:57,351] INFO : Step 37000/500000 rank0 | Train Loss: 8.4942 | Train Loss Unweighted: 3.7323 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.2968 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1807 | Regression_eta: 0.0107 | Regression_pt: 0.1894 | Regression_sin_phi: 0.0136 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3518 | Classification_binary: 3.2968 | Regression_cos_phi: 0.1071 | Regression_energy: 1.9248 | Regression_eta: 0.0858 | Regression_pt: 2.5812 | Regression_sin_phi: 0.1467 | LR: 8.85e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4784s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0027 | max_backward: 0.0403 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1169 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2854.0 | uss_MB: 1786.5 | child_rss_MB: 22491.9 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:44:07,646] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 06:44:07,646] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:45:11,086] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 06:45:11,086] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:45:49,940] INFO : Step 38000/500000 rank0 | Train Loss: 8.2303 | Train Loss Unweighted: 3.8784 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.4812 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1722 | Regression_eta: 0.0067 | Regression_pt: 0.1734 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3159 | Classification_binary: 3.4812 | Regression_cos_phi: 0.0959 | Regression_energy: 1.8350 | Regression_eta: 0.0543 | Regression_pt: 2.3633 | Regression_sin_phi: 0.0847 | LR: 9.21e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4964s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0032 | max_backward: 0.0415 | max_optimizer_step: 0.0642 | max_tensorboard_logging: 0.1245 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2851.5 | uss_MB: 1784.0 | child_rss_MB: 22493.4 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:45:49,942] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:45:50,014] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:46:14,148] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 06:46:14,148] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:16,885] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 06:47:16,885] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:42,229] INFO : Step 39000/500000 rank0 | Train Loss: 8.7446 | Train Loss Unweighted: 3.9702 | Train Loss Components: Classification: 0.0286 | Classification_binary: 3.5341 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1846 | Regression_eta: 0.0087 | Regression_pt: 0.1906 | Regression_sin_phi: 0.0132 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3194 | Classification_binary: 3.5341 | Regression_cos_phi: 0.1153 | Regression_energy: 1.9665 | Regression_eta: 0.0703 | Regression_pt: 2.5967 | Regression_sin_phi: 0.1421 | LR: 9.57e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4976s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0412 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.1233 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2854.6 | uss_MB: 1787.2 | child_rss_MB: 22499.7 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:48:20,077] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 06:48:20,077] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:22,873] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 06:49:22,873] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:34,191] INFO : Step 40000/500000 rank0 | Train Loss: 7.5032 | Train Loss Unweighted: 3.5061 | Train Loss Components: Classification: 0.0201 | Classification_binary: 3.1409 | Regression_cos_phi: 0.0107 | Regression_energy: 0.1564 | Regression_eta: 0.0082 | Regression_pt: 0.1602 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2244 | Classification_binary: 3.1409 | Regression_cos_phi: 0.1183 | Regression_energy: 1.6661 | Regression_eta: 0.0660 | Regression_pt: 2.1828 | Regression_sin_phi: 0.1048 | LR: 9.94e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4891s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0407 | max_optimizer_step: 0.0648 | max_tensorboard_logging: 0.1216 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2855.0 | uss_MB: 1787.6 | child_rss_MB: 22498.8 | allocated_MB: 125.7 | reserved_MB: 632.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:49:34,193] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 06:49:34,265] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:49:34,266] INFO : Running validation on rank0 +rank0 [2026-11-14 06:49:34,266] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:49:34,266] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:49:34,416] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 06:49:34,536] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:49:34,655] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:49:34,727] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:49:34,795] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:49:34,863] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:49:34,933] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:49:35,003] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:49:35,124] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:49:35,244] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:49:35,361] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:49:35,430] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:49:35,497] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:49:35,566] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:49:35,633] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:49:35,701] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:49:35,820] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:49:35,828] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:49:35,830] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:49:35,832] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:49:35,837] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:49:35,839] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:49:35,841] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:49:35,843] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:49:35,844] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:49:35,847] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:49:35,849] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:49:35,851] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:49:35,853] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:49:35,855] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:49:35,857] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:49:35,859] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:49:37,445] INFO : VALIDATION | Step=40000/500000 | Train Loss=7.5032 | Valid Loss=8.5265 | Train Loss Unweighted=3.5061 | Valid Loss Unweighted=3.8310 | Stale=0 | ETA=872.1m +rank0 [2026-11-14 06:49:37,446] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:49:38,146] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:49:38,154] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:38,197] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:49:38,716] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:49:38,981] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:49:39,240] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:49:39,499] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:49:39,758] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:49:40,013] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:49:40,274] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:49:40,530] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:49:40,790] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:49:41,042] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:49:41,293] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:49:41,546] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:49:41,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:49:42,058] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:49:42,327] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:49:42,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:49:42,616] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:49:42,634] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:43,208] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:43,593] INFO : Plotted number of elements +rank0 [2026-11-14 06:49:44,978] INFO : Plotted jets +rank0 [2026-11-14 06:49:45,992] INFO : Plotted jet ratio +rank0 [2026-11-14 06:50:38,725] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 06:50:38,725] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:51:39,715] INFO : Step 41000/500000 rank0 | Train Loss: 6.9763 | Train Loss Unweighted: 3.5328 | Train Loss Components: Classification: 0.0236 | Classification_binary: 3.2180 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1332 | Regression_eta: 0.0058 | Regression_pt: 0.1356 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2638 | Classification_binary: 3.2180 | Regression_cos_phi: 0.0972 | Regression_energy: 1.4191 | Regression_eta: 0.0465 | Regression_pt: 1.8481 | Regression_sin_phi: 0.0835 | LR: 1.03e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5217s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0378 | max_loss: 0.0028 | max_backward: 0.0416 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.1369 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2875.1 | uss_MB: 1807.7 | child_rss_MB: 22499.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:51:42,724] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 06:51:42,724] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:52:46,683] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 06:52:46,683] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:53:33,882] INFO : Step 42000/500000 rank0 | Train Loss: 7.3093 | Train Loss Unweighted: 3.4774 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.1213 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1385 | Regression_eta: 0.0219 | Regression_pt: 0.1448 | Regression_sin_phi: 0.0144 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3326 | Classification_binary: 3.1213 | Regression_cos_phi: 0.0760 | Regression_energy: 1.4753 | Regression_eta: 0.1759 | Regression_pt: 1.9730 | Regression_sin_phi: 0.1552 | LR: 1.07e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5197s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0389 | max_loss: 0.0032 | max_backward: 0.0434 | max_optimizer_step: 0.0679 | max_tensorboard_logging: 0.1217 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2875.8 | uss_MB: 1808.4 | child_rss_MB: 22502.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 24.3 +rank0 [2026-11-14 06:53:33,884] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:53:33,956] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:53:50,964] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 06:53:50,964] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:54:55,101] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 06:54:55,101] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:55:28,197] INFO : Step 43000/500000 rank0 | Train Loss: 7.8496 | Train Loss Unweighted: 3.6933 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.3133 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1618 | Regression_eta: 0.0057 | Regression_pt: 0.1620 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3340 | Classification_binary: 3.3133 | Regression_cos_phi: 0.1049 | Regression_energy: 1.7242 | Regression_eta: 0.0461 | Regression_pt: 2.2077 | Regression_sin_phi: 0.1194 | LR: 1.11e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4960s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0637 | max_tensorboard_logging: 0.1213 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2878.7 | uss_MB: 1811.3 | child_rss_MB: 22505.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:55:59,193] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 06:55:59,193] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:57:02,551] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 06:57:02,551] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:57:21,561] INFO : Step 44000/500000 rank0 | Train Loss: 9.3359 | Train Loss Unweighted: 4.0338 | Train Loss Components: Classification: 0.0265 | Classification_binary: 3.5511 | Regression_cos_phi: 0.0081 | Regression_energy: 0.2164 | Regression_eta: 0.0085 | Regression_pt: 0.2173 | Regression_sin_phi: 0.0061 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2955 | Classification_binary: 3.5511 | Regression_cos_phi: 0.0889 | Regression_energy: 2.3057 | Regression_eta: 0.0682 | Regression_pt: 2.9606 | Regression_sin_phi: 0.0660 | LR: 1.15e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4950s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1221 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2879.1 | uss_MB: 1811.7 | child_rss_MB: 22507.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:57:21,563] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:57:21,634] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:58:06,743] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 06:58:06,743] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:59:10,714] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 06:59:10,715] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:59:15,625] INFO : Step 45000/500000 rank0 | Train Loss: 7.4542 | Train Loss Unweighted: 3.6780 | Train Loss Components: Classification: 0.0311 | Classification_binary: 3.3315 | Regression_cos_phi: 0.0075 | Regression_energy: 0.1439 | Regression_eta: 0.0082 | Regression_pt: 0.1455 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3470 | Classification_binary: 3.3315 | Regression_cos_phi: 0.0825 | Regression_energy: 1.5333 | Regression_eta: 0.0659 | Regression_pt: 1.9830 | Regression_sin_phi: 0.1109 | LR: 1.19e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4977s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0027 | max_backward: 0.0441 | max_optimizer_step: 0.0671 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2879.2 | uss_MB: 1811.8 | child_rss_MB: 22505.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:00:14,596] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 07:00:14,596] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:01:07,453] INFO : Step 46000/500000 rank0 | Train Loss: 7.2933 | Train Loss Unweighted: 3.5687 | Train Loss Components: Classification: 0.0266 | Classification_binary: 3.2281 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1432 | Regression_eta: 0.0062 | Regression_pt: 0.1462 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2974 | Classification_binary: 3.2281 | Regression_cos_phi: 0.1068 | Regression_energy: 1.5255 | Regression_eta: 0.0499 | Regression_pt: 1.9917 | Regression_sin_phi: 0.0939 | LR: 1.23e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5375s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0385 | max_loss: 0.0032 | max_backward: 0.0470 | max_optimizer_step: 0.0709 | max_tensorboard_logging: 0.1261 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2875.9 | uss_MB: 1808.5 | child_rss_MB: 22507.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:01:07,455] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:01:07,529] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 07:01:17,430] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 07:01:17,430] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:02:22,231] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 07:02:22,231] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:03:00,939] INFO : Step 47000/500000 rank0 | Train Loss: 7.8079 | Train Loss Unweighted: 3.7243 | Train Loss Components: Classification: 0.0346 | Classification_binary: 3.3497 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1526 | Regression_eta: 0.0079 | Regression_pt: 0.1563 | Regression_sin_phi: 0.0126 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3866 | Classification_binary: 3.3497 | Regression_cos_phi: 0.1163 | Regression_energy: 1.6255 | Regression_eta: 0.0638 | Regression_pt: 2.1296 | Regression_sin_phi: 0.1365 | LR: 1.27e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4937s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0027 | max_backward: 0.0395 | max_optimizer_step: 0.0627 | max_tensorboard_logging: 0.1352 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2878.7 | uss_MB: 1811.3 | child_rss_MB: 22512.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:03:23,519] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 07:03:23,519] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:27,528] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 07:04:27,528] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:53,830] INFO : Step 48000/500000 rank0 | Train Loss: 7.0421 | Train Loss Unweighted: 3.5633 | Train Loss Components: Classification: 0.0320 | Classification_binary: 3.2445 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1342 | Regression_eta: 0.0064 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3570 | Classification_binary: 3.2445 | Regression_cos_phi: 0.0769 | Regression_energy: 1.4301 | Regression_eta: 0.0512 | Regression_pt: 1.8214 | Regression_sin_phi: 0.0610 | LR: 1.31e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5114s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0005 | max_forward: 0.0371 | max_loss: 0.0030 | max_backward: 0.0426 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1278 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2878.8 | uss_MB: 1811.4 | child_rss_MB: 22510.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:04:53,832] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:04:53,913] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 07:05:31,568] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 07:05:31,568] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:35,219] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 07:06:35,220] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:47,607] INFO : Step 49000/500000 rank0 | Train Loss: 7.1939 | Train Loss Unweighted: 3.5544 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.2200 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1411 | Regression_eta: 0.0113 | Regression_pt: 0.1424 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3028 | Classification_binary: 3.2200 | Regression_cos_phi: 0.0685 | Regression_energy: 1.5029 | Regression_eta: 0.0913 | Regression_pt: 1.9407 | Regression_sin_phi: 0.0676 | LR: 1.36e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5107s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0027 | max_backward: 0.0418 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2878.8 | uss_MB: 1811.4 | child_rss_MB: 22508.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:07:39,224] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 07:07:39,224] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:08:41,446] INFO : Step 50000/500000 rank0 | Train Loss: 7.9581 | Train Loss Unweighted: 3.5685 | Train Loss Components: Classification: 0.0316 | Classification_binary: 3.1659 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1726 | Regression_eta: 0.0088 | Regression_pt: 0.1692 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3524 | Classification_binary: 3.1659 | Regression_cos_phi: 0.1225 | Regression_energy: 1.8386 | Regression_eta: 0.0705 | Regression_pt: 2.3054 | Regression_sin_phi: 0.1029 | LR: 1.40e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5013s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0030 | max_backward: 0.0407 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1222 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2875.3 | uss_MB: 1807.9 | child_rss_MB: 22512.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:08:41,448] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:08:41,517] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 07:08:41,518] INFO : Running validation on rank0 +rank0 [2026-11-14 07:08:41,518] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:08:41,518] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:08:41,668] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 07:08:41,795] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:08:41,915] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:08:41,986] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:08:42,055] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:08:42,125] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:08:42,199] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:08:42,269] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:08:42,374] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:08:42,494] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:08:42,616] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:08:42,687] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:08:42,758] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:08:42,825] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:08:42,895] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:08:42,964] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:08:43,069] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:08:43,076] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:08:43,079] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:08:43,081] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:08:43,083] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:08:43,083] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:08:43,087] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:08:43,089] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:08:43,091] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:08:43,093] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:08:43,095] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:08:43,097] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:08:43,099] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:08:43,101] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:08:43,103] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:08:43,105] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:08:44,698] INFO : VALIDATION | Step=50000/500000 | Train Loss=7.9581 | Valid Loss=8.4101 | Train Loss Unweighted=3.5685 | Valid Loss Unweighted=3.7656 | Stale=0 | ETA=854.6m +rank0 [2026-11-14 07:08:44,698] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:08:45,074] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:08:45,109] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:45,158] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:08:45,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:08:45,928] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:08:46,190] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:08:46,449] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:08:46,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:08:46,984] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:08:47,242] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:08:47,502] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:08:47,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:08:48,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:08:48,279] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:08:48,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:08:48,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:08:49,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:08:49,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:08:49,542] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:08:49,605] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:08:49,637] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:50,254] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:50,649] INFO : Plotted number of elements +rank0 [2026-11-14 07:08:52,021] INFO : Plotted jets +rank0 [2026-11-14 07:08:53,037] INFO : Plotted jet ratio +rank0 [2026-11-14 07:08:55,562] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 07:08:55,562] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:09:58,946] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 07:09:58,946] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:10:46,790] INFO : Step 51000/500000 rank0 | Train Loss: 7.2487 | Train Loss Unweighted: 3.5729 | Train Loss Components: Classification: 0.0288 | Classification_binary: 3.2342 | Regression_cos_phi: 0.0098 | Regression_energy: 0.1389 | Regression_eta: 0.0105 | Regression_pt: 0.1389 | Regression_sin_phi: 0.0117 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3218 | Classification_binary: 3.2342 | Regression_cos_phi: 0.1083 | Regression_energy: 1.4799 | Regression_eta: 0.0847 | Regression_pt: 1.8928 | Regression_sin_phi: 0.1269 | LR: 1.44e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4946s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0028 | max_backward: 0.0393 | max_optimizer_step: 0.0674 | max_tensorboard_logging: 0.1192 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3007.4 | uss_MB: 1940.1 | child_rss_MB: 22515.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:11:02,852] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 07:11:02,852] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:12:06,013] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 07:12:06,014] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:12:39,691] INFO : Step 52000/500000 rank0 | Train Loss: 6.7873 | Train Loss Unweighted: 3.5016 | Train Loss Components: Classification: 0.0340 | Classification_binary: 3.1996 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1214 | Regression_eta: 0.0057 | Regression_pt: 0.1220 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3794 | Classification_binary: 3.1996 | Regression_cos_phi: 0.0943 | Regression_energy: 1.2940 | Regression_eta: 0.0459 | Regression_pt: 1.6624 | Regression_sin_phi: 0.1117 | LR: 1.49e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5084s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0030 | max_backward: 0.0420 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1226 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3007.5 | uss_MB: 1940.1 | child_rss_MB: 22516.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:12:39,693] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:12:39,758] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:13:09,719] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 07:13:09,719] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:14:13,175] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 07:14:13,175] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:14:32,974] INFO : Step 53000/500000 rank0 | Train Loss: 7.2691 | Train Loss Unweighted: 3.7245 | Train Loss Components: Classification: 0.0386 | Classification_binary: 3.3986 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1309 | Regression_eta: 0.0085 | Regression_pt: 0.1333 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.4303 | Classification_binary: 3.3986 | Regression_cos_phi: 0.0932 | Regression_energy: 1.3951 | Regression_eta: 0.0685 | Regression_pt: 1.8158 | Regression_sin_phi: 0.0676 | LR: 1.53e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5154s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0031 | max_backward: 0.0411 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1249 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3008.2 | uss_MB: 1940.8 | child_rss_MB: 22516.4 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 74.3 +rank0 [2026-11-14 07:15:16,840] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 07:15:16,840] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:20,073] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 07:16:20,073] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:25,861] INFO : Step 54000/500000 rank0 | Train Loss: 6.3539 | Train Loss Unweighted: 3.3410 | Train Loss Components: Classification: 0.0292 | Classification_binary: 3.0633 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1131 | Regression_eta: 0.0095 | Regression_pt: 0.1140 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3256 | Classification_binary: 3.0633 | Regression_cos_phi: 0.0694 | Regression_energy: 1.2055 | Regression_eta: 0.0763 | Regression_pt: 1.5534 | Regression_sin_phi: 0.0603 | LR: 1.58e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4924s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0030 | max_backward: 0.0406 | max_optimizer_step: 0.0665 | max_tensorboard_logging: 0.1200 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2901.0 | uss_MB: 1833.6 | child_rss_MB: 22516.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:16:25,863] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:16:25,930] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:17:23,494] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 07:17:23,494] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:18:18,685] INFO : Step 55000/500000 rank0 | Train Loss: 6.6991 | Train Loss Unweighted: 3.5704 | Train Loss Components: Classification: 0.0241 | Classification_binary: 3.2818 | Regression_cos_phi: 0.0084 | Regression_energy: 0.1181 | Regression_eta: 0.0085 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0124 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2694 | Classification_binary: 3.2818 | Regression_cos_phi: 0.0927 | Regression_energy: 1.2578 | Regression_eta: 0.0686 | Regression_pt: 1.5948 | Regression_sin_phi: 0.1340 | LR: 1.62e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.4959s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0376 | max_loss: 0.0030 | max_backward: 0.0419 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1207 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2903.4 | uss_MB: 1836.0 | child_rss_MB: 22521.7 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:18:27,223] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 07:18:27,224] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:19:30,315] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 07:19:30,315] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:20:11,285] INFO : Step 56000/500000 rank0 | Train Loss: 6.9033 | Train Loss Unweighted: 3.4739 | Train Loss Components: Classification: 0.0312 | Classification_binary: 3.1586 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1323 | Regression_eta: 0.0068 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3477 | Classification_binary: 3.1586 | Regression_cos_phi: 0.1076 | Regression_energy: 1.4099 | Regression_eta: 0.0545 | Regression_pt: 1.7567 | Regression_sin_phi: 0.0683 | LR: 1.67e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5026s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0030 | max_backward: 0.0406 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1281 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2903.7 | uss_MB: 1836.3 | child_rss_MB: 22521.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:20:11,287] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:20:11,355] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:20:33,702] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 07:20:33,703] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:21:36,858] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 07:21:36,858] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:22:04,178] INFO : Step 57000/500000 rank0 | Train Loss: 5.9924 | Train Loss Unweighted: 3.1726 | Train Loss Components: Classification: 0.0306 | Classification_binary: 2.9128 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1074 | Regression_eta: 0.0063 | Regression_pt: 0.1043 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3419 | Classification_binary: 2.9128 | Regression_cos_phi: 0.0576 | Regression_energy: 1.1440 | Regression_eta: 0.0509 | Regression_pt: 1.4209 | Regression_sin_phi: 0.0643 | LR: 1.72e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5087s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0369 | max_loss: 0.0029 | max_backward: 0.0409 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1309 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2903.7 | uss_MB: 1836.3 | child_rss_MB: 22517.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:22:41,039] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 07:22:41,039] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:23:44,523] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 07:23:44,523] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:23:57,835] INFO : Step 58000/500000 rank0 | Train Loss: 5.8144 | Train Loss Unweighted: 3.1276 | Train Loss Components: Classification: 0.0309 | Classification_binary: 2.8801 | Regression_cos_phi: 0.0079 | Regression_energy: 0.0973 | Regression_eta: 0.0064 | Regression_pt: 0.0991 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3443 | Classification_binary: 2.8801 | Regression_cos_phi: 0.0867 | Regression_energy: 1.0366 | Regression_eta: 0.0512 | Regression_pt: 1.3504 | Regression_sin_phi: 0.0651 | LR: 1.76e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.4970s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0407 | max_loss: 0.0029 | max_backward: 0.0410 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1227 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2900.4 | uss_MB: 1833.0 | child_rss_MB: 22519.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:23:57,837] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:23:57,907] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:24:48,615] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 07:24:48,615] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:25:51,076] INFO : Step 59000/500000 rank0 | Train Loss: 7.3318 | Train Loss Unweighted: 3.3716 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.0056 | Regression_cos_phi: 0.0206 | Regression_energy: 0.1458 | Regression_eta: 0.0133 | Regression_pt: 0.1473 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3118 | Classification_binary: 3.0056 | Regression_cos_phi: 0.2276 | Regression_energy: 1.5534 | Regression_eta: 0.1070 | Regression_pt: 2.0076 | Regression_sin_phi: 0.1188 | LR: 1.81e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.4959s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0371 | max_loss: 0.0027 | max_backward: 0.0410 | max_optimizer_step: 0.0648 | max_tensorboard_logging: 0.1251 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2902.6 | uss_MB: 1835.2 | child_rss_MB: 22520.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:25:52,194] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 07:25:52,194] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:26:55,412] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 07:26:55,413] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:27:44,653] INFO : Step 60000/500000 rank0 | Train Loss: 6.2471 | Train Loss Unweighted: 3.3897 | Train Loss Components: Classification: 0.0222 | Classification_binary: 3.1248 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1079 | Regression_eta: 0.0123 | Regression_pt: 0.1064 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2481 | Classification_binary: 3.1248 | Regression_cos_phi: 0.0883 | Regression_energy: 1.1498 | Regression_eta: 0.0993 | Regression_pt: 1.4495 | Regression_sin_phi: 0.0873 | LR: 1.86e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.4936s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0374 | max_loss: 0.0032 | max_backward: 0.0421 | max_optimizer_step: 0.0661 | max_tensorboard_logging: 0.1188 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2902.8 | uss_MB: 1835.4 | child_rss_MB: 22517.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:27:44,655] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:27:44,725] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:27:44,726] INFO : Running validation on rank0 +rank0 [2026-11-14 07:27:44,726] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:27:44,726] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:27:44,875] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 07:27:45,001] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:27:45,122] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:27:45,193] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:27:45,260] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:27:45,326] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:27:45,395] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:27:45,465] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:27:45,582] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:27:45,702] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:27:45,820] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:27:45,889] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:27:45,956] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:27:46,023] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:27:46,092] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:27:46,161] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:27:46,264] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:27:46,271] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:27:46,273] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:27:46,276] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:27:46,278] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:27:46,280] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:27:46,282] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:27:46,284] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:27:46,286] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:27:46,288] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:27:46,290] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:27:46,292] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:27:46,294] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:27:46,296] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:27:46,297] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:27:46,300] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:27:47,822] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.2471 | Valid Loss=8.7496 | Train Loss Unweighted=3.3897 | Valid Loss Unweighted=3.7868 | Stale=1 | ETA=836.1m +rank0 [2026-11-14 07:27:47,822] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:27:48,161] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:27:48,170] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:48,215] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:27:48,720] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:27:48,991] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:27:49,246] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:27:49,503] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:27:49,760] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:27:50,015] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:27:50,269] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:27:50,521] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:27:50,782] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:27:51,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:27:51,312] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:27:51,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:27:51,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:27:52,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:27:52,354] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:27:52,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:27:52,655] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:27:52,661] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:53,251] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:53,633] INFO : Plotted number of elements +rank0 [2026-11-14 07:27:54,993] INFO : Plotted jets +rank0 [2026-11-14 07:27:55,986] INFO : Plotted jet ratio +rank0 [2026-11-14 07:28:11,530] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 07:28:11,530] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:13,135] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 07:29:13,135] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:48,893] INFO : Step 61000/500000 rank0 | Train Loss: 6.3731 | Train Loss Unweighted: 3.5179 | Train Loss Components: Classification: 0.0259 | Classification_binary: 3.2543 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1071 | Regression_eta: 0.0068 | Regression_pt: 0.1040 | Regression_sin_phi: 0.0084 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2893 | Classification_binary: 3.2543 | Regression_cos_phi: 0.1254 | Regression_energy: 1.1413 | Regression_eta: 0.0551 | Regression_pt: 1.4167 | Regression_sin_phi: 0.0911 | LR: 1.91e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5294s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0034 | max_backward: 0.0431 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.1258 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3027.7 | uss_MB: 1960.3 | child_rss_MB: 22517.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 28.3 +rank0 [2026-11-14 07:30:19,624] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 07:30:19,624] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:31:21,144] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 07:31:21,144] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:31:41,226] INFO : Step 62000/500000 rank0 | Train Loss: 6.7947 | Train Loss Unweighted: 3.4449 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.1382 | Regression_cos_phi: 0.0044 | Regression_energy: 0.1276 | Regression_eta: 0.0083 | Regression_pt: 0.1325 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2932 | Classification_binary: 3.1382 | Regression_cos_phi: 0.0488 | Regression_energy: 1.3593 | Regression_eta: 0.0667 | Regression_pt: 1.8054 | Regression_sin_phi: 0.0832 | LR: 1.95e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5009s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0374 | max_loss: 0.0032 | max_backward: 0.0413 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1191 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2926.1 | uss_MB: 1858.7 | child_rss_MB: 22517.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:31:41,228] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:31:41,298] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:32:24,992] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 07:32:24,992] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:33:29,004] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 07:33:29,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:33:35,798] INFO : Step 63000/500000 rank0 | Train Loss: 6.4852 | Train Loss Unweighted: 3.3940 | Train Loss Components: Classification: 0.0194 | Classification_binary: 3.1096 | Regression_cos_phi: 0.0101 | Regression_energy: 0.1172 | Regression_eta: 0.0084 | Regression_pt: 0.1184 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2166 | Classification_binary: 3.1096 | Regression_cos_phi: 0.1112 | Regression_energy: 1.2491 | Regression_eta: 0.0679 | Regression_pt: 1.6128 | Regression_sin_phi: 0.1180 | LR: 2.00e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5138s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0394 | max_loss: 0.0031 | max_backward: 0.0461 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.1236 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2929.6 | uss_MB: 1862.2 | child_rss_MB: 22524.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:34:33,337] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 07:34:33,337] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:35:30,283] INFO : Step 64000/500000 rank0 | Train Loss: 6.0350 | Train Loss Unweighted: 3.2915 | Train Loss Components: Classification: 0.0250 | Classification_binary: 3.0390 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1003 | Regression_eta: 0.0073 | Regression_pt: 0.1035 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2789 | Classification_binary: 3.0390 | Regression_cos_phi: 0.1172 | Regression_energy: 1.0687 | Regression_eta: 0.0588 | Regression_pt: 1.4109 | Regression_sin_phi: 0.0614 | LR: 2.05e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5254s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0402 | max_loss: 0.0031 | max_backward: 0.0407 | max_optimizer_step: 0.0673 | max_tensorboard_logging: 0.1365 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2929.7 | uss_MB: 1862.3 | child_rss_MB: 22519.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:35:30,285] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:35:30,358] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:35:38,112] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 07:35:38,112] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:36:42,256] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 07:36:42,257] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:37:25,033] INFO : Step 65000/500000 rank0 | Train Loss: 6.5617 | Train Loss Unweighted: 3.3415 | Train Loss Components: Classification: 0.0231 | Classification_binary: 3.0467 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1290 | Regression_eta: 0.0052 | Regression_pt: 0.1255 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2574 | Classification_binary: 3.0467 | Regression_cos_phi: 0.0590 | Regression_energy: 1.3744 | Regression_eta: 0.0419 | Regression_pt: 1.7105 | Regression_sin_phi: 0.0718 | LR: 2.10e-04 | DataLoad Time: 0.0118s | Model Forward Time: 0.5157s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0384 | max_loss: 0.0032 | max_backward: 0.0426 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2929.9 | uss_MB: 1862.5 | child_rss_MB: 22520.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:37:46,702] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 07:37:46,702] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:38:50,423] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 07:38:50,423] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:39:18,677] INFO : Step 66000/500000 rank0 | Train Loss: 6.4530 | Train Loss Unweighted: 3.2832 | Train Loss Components: Classification: 0.0290 | Classification_binary: 2.9930 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1181 | Regression_eta: 0.0044 | Regression_pt: 0.1209 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3237 | Classification_binary: 2.9930 | Regression_cos_phi: 0.1270 | Regression_energy: 1.2581 | Regression_eta: 0.0352 | Regression_pt: 1.6476 | Regression_sin_phi: 0.0684 | LR: 2.15e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5049s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0407 | max_loss: 0.0031 | max_backward: 0.0410 | max_optimizer_step: 0.0655 | max_tensorboard_logging: 0.1231 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2925.5 | uss_MB: 1858.1 | child_rss_MB: 22521.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:39:18,678] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:39:18,751] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:39:54,418] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 07:39:54,419] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:40:57,911] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 07:40:57,911] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:41:12,226] INFO : Step 67000/500000 rank0 | Train Loss: 5.7913 | Train Loss Unweighted: 3.2797 | Train Loss Components: Classification: 0.0245 | Classification_binary: 3.0458 | Regression_cos_phi: 0.0148 | Regression_energy: 0.0887 | Regression_eta: 0.0095 | Regression_pt: 0.0872 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2738 | Classification_binary: 3.0458 | Regression_cos_phi: 0.1635 | Regression_energy: 0.9453 | Regression_eta: 0.0765 | Regression_pt: 1.1883 | Regression_sin_phi: 0.0982 | LR: 2.20e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5182s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0384 | max_loss: 0.0028 | max_backward: 0.0417 | max_optimizer_step: 0.0635 | max_tensorboard_logging: 0.1375 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2929.1 | uss_MB: 1861.8 | child_rss_MB: 22528.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:42:02,452] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 07:42:02,452] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:43:06,550] INFO : Step 68000/500000 rank0 | Train Loss: 5.6031 | Train Loss Unweighted: 3.2827 | Train Loss Components: Classification: 0.0206 | Classification_binary: 3.0673 | Regression_cos_phi: 0.0137 | Regression_energy: 0.0836 | Regression_eta: 0.0075 | Regression_pt: 0.0819 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2299 | Classification_binary: 3.0673 | Regression_cos_phi: 0.1512 | Regression_energy: 0.8903 | Regression_eta: 0.0604 | Regression_pt: 1.1165 | Regression_sin_phi: 0.0874 | LR: 2.25e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5079s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0397 | max_loss: 0.0031 | max_backward: 0.0423 | max_optimizer_step: 0.0655 | max_tensorboard_logging: 0.1226 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2929.3 | uss_MB: 1861.9 | child_rss_MB: 22527.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:43:06,552] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:43:06,622] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:43:06,846] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 07:43:06,846] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:44:10,801] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 07:44:10,802] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:45:00,571] INFO : Step 69000/500000 rank0 | Train Loss: 5.4339 | Train Loss Unweighted: 2.9482 | Train Loss Components: Classification: 0.0281 | Classification_binary: 2.7169 | Regression_cos_phi: 0.0106 | Regression_energy: 0.0901 | Regression_eta: 0.0090 | Regression_pt: 0.0864 | Regression_sin_phi: 0.0071 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3133 | Classification_binary: 2.7169 | Regression_cos_phi: 0.1170 | Regression_energy: 0.9595 | Regression_eta: 0.0727 | Regression_pt: 1.1776 | Regression_sin_phi: 0.0768 | LR: 2.30e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4937s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0031 | max_backward: 0.0412 | max_optimizer_step: 0.0647 | max_tensorboard_logging: 0.1254 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2929.4 | uss_MB: 1862.0 | child_rss_MB: 22528.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:45:14,781] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 07:45:14,782] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:46:18,409] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 07:46:18,409] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:46:54,544] INFO : Step 70000/500000 rank0 | Train Loss: 6.2062 | Train Loss Unweighted: 3.2431 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.9678 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1119 | Regression_eta: 0.0130 | Regression_pt: 0.1084 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2756 | Classification_binary: 2.9678 | Regression_cos_phi: 0.1006 | Regression_energy: 1.1927 | Regression_eta: 0.1046 | Regression_pt: 1.4770 | Regression_sin_phi: 0.0879 | LR: 2.35e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5101s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0029 | max_backward: 0.0429 | max_optimizer_step: 0.0689 | max_tensorboard_logging: 0.1238 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2925.5 | uss_MB: 1858.1 | child_rss_MB: 22531.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:46:54,546] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:46:54,615] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:46:54,616] INFO : Running validation on rank0 +rank0 [2026-11-14 07:46:54,616] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:46:54,616] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:46:54,765] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 07:46:54,887] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:46:55,003] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:46:55,075] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:46:55,142] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:46:55,209] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:46:55,279] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:46:55,349] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:46:55,448] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:46:55,568] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:46:55,686] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:46:55,756] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:46:55,825] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:46:55,893] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:46:55,961] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:46:56,029] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:46:56,135] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:46:56,142] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:46:56,145] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:46:56,147] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:46:56,149] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:46:56,151] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:46:56,153] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:46:56,155] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:46:56,157] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:46:56,159] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:46:56,161] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:46:56,163] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:46:56,166] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:46:56,168] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:46:56,170] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:46:56,172] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:46:57,688] INFO : VALIDATION | Step=70000/500000 | Train Loss=6.2062 | Valid Loss=9.2872 | Train Loss Unweighted=3.2431 | Valid Loss Unweighted=3.9482 | Stale=2 | ETA=818.1m +rank0 [2026-11-14 07:46:57,689] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:46:58,199] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:46:58,206] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:46:58,251] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:46:58,775] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:46:59,034] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:46:59,291] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:46:59,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:46:59,801] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:47:00,059] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:47:00,313] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:47:00,579] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:47:00,841] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:47:01,097] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:47:01,352] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:47:01,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:47:01,870] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:47:02,122] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:47:02,383] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:47:02,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:47:02,667] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:47:02,674] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:47:03,272] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:47:03,661] INFO : Plotted number of elements +rank0 [2026-11-14 07:47:05,017] INFO : Plotted jets +rank0 [2026-11-14 07:47:06,023] INFO : Plotted jet ratio +rank0 [2026-11-14 07:47:34,815] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 07:47:34,815] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:48:38,232] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 07:48:38,232] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:49:00,168] INFO : Step 71000/500000 rank0 | Train Loss: 5.2992 | Train Loss Unweighted: 2.9020 | Train Loss Components: Classification: 0.0248 | Classification_binary: 2.6804 | Regression_cos_phi: 0.0073 | Regression_energy: 0.0871 | Regression_eta: 0.0079 | Regression_pt: 0.0879 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2762 | Classification_binary: 2.6804 | Regression_cos_phi: 0.0806 | Regression_energy: 0.9282 | Regression_eta: 0.0637 | Regression_pt: 1.1982 | Regression_sin_phi: 0.0720 | LR: 2.40e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5202s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0028 | max_backward: 0.0400 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1377 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3058.0 | uss_MB: 1990.6 | child_rss_MB: 22530.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:49:42,426] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 07:49:42,427] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:50:45,479] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 07:50:45,479] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:50:53,134] INFO : Step 72000/500000 rank0 | Train Loss: 5.2080 | Train Loss Unweighted: 3.1557 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.9655 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0744 | Regression_eta: 0.0077 | Regression_pt: 0.0747 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2393 | Classification_binary: 2.9655 | Regression_cos_phi: 0.0662 | Regression_energy: 0.7930 | Regression_eta: 0.0616 | Regression_pt: 1.0178 | Regression_sin_phi: 0.0646 | LR: 2.45e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5100s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0030 | max_backward: 0.0422 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1213 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3058.0 | uss_MB: 1990.6 | child_rss_MB: 22536.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:50:53,136] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 07:50:53,206] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:51:49,403] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 07:51:49,403] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:52:46,634] INFO : Step 73000/500000 rank0 | Train Loss: 6.1049 | Train Loss Unweighted: 3.0808 | Train Loss Components: Classification: 0.0293 | Classification_binary: 2.7993 | Regression_cos_phi: 0.0228 | Regression_energy: 0.1027 | Regression_eta: 0.0119 | Regression_pt: 0.1052 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.3269 | Classification_binary: 2.7993 | Regression_cos_phi: 0.2522 | Regression_energy: 1.0938 | Regression_eta: 0.0958 | Regression_pt: 1.4329 | Regression_sin_phi: 0.1041 | LR: 2.50e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5163s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0031 | max_backward: 0.0407 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1273 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3058.0 | uss_MB: 1990.6 | child_rss_MB: 22535.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:52:53,427] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 07:52:53,427] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:53:56,875] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 07:53:56,875] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:54:40,097] INFO : Step 74000/500000 rank0 | Train Loss: 5.6216 | Train Loss Unweighted: 3.1573 | Train Loss Components: Classification: 0.0232 | Classification_binary: 2.9315 | Regression_cos_phi: 0.0061 | Regression_energy: 0.0935 | Regression_eta: 0.0034 | Regression_pt: 0.0935 | Regression_sin_phi: 0.0062 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2586 | Classification_binary: 2.9315 | Regression_cos_phi: 0.0675 | Regression_energy: 0.9959 | Regression_eta: 0.0278 | Regression_pt: 1.2736 | Regression_sin_phi: 0.0667 | LR: 2.55e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5174s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0405 | max_optimizer_step: 0.0658 | max_tensorboard_logging: 0.1287 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2950.7 | uss_MB: 1883.3 | child_rss_MB: 22535.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:54:40,099] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 07:54:40,172] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:55:00,975] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 07:55:00,975] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:56:04,289] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 07:56:04,289] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:56:33,908] INFO : Step 75000/500000 rank0 | Train Loss: 5.6161 | Train Loss Unweighted: 2.8479 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.5940 | Regression_cos_phi: 0.0081 | Regression_energy: 0.1043 | Regression_eta: 0.0059 | Regression_pt: 0.1066 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2371 | Classification_binary: 2.5940 | Regression_cos_phi: 0.0895 | Regression_energy: 1.1109 | Regression_eta: 0.0471 | Regression_pt: 1.4530 | Regression_sin_phi: 0.0844 | LR: 2.60e-04 | DataLoad Time: 0.0055s | Model Forward Time: 0.5153s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0392 | max_loss: 0.0030 | max_backward: 0.0416 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1304 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2953.2 | uss_MB: 1885.9 | child_rss_MB: 22533.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:57:07,870] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 07:57:07,870] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:58:11,414] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 07:58:11,414] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:58:27,159] INFO : Step 76000/500000 rank0 | Train Loss: 4.8464 | Train Loss Unweighted: 2.6441 | Train Loss Components: Classification: 0.0239 | Classification_binary: 2.4399 | Regression_cos_phi: 0.0083 | Regression_energy: 0.0787 | Regression_eta: 0.0086 | Regression_pt: 0.0800 | Regression_sin_phi: 0.0047 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2666 | Classification_binary: 2.4399 | Regression_cos_phi: 0.0914 | Regression_energy: 0.8388 | Regression_eta: 0.0691 | Regression_pt: 1.0897 | Regression_sin_phi: 0.0508 | LR: 2.65e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5566s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0005 | max_forward: 0.0392 | max_loss: 0.0037 | max_backward: 0.0427 | max_optimizer_step: 0.0719 | max_tensorboard_logging: 0.1284 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2953.6 | uss_MB: 1886.2 | child_rss_MB: 22538.8 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:58:27,161] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 07:58:27,231] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:59:15,431] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 07:59:15,432] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:17,527] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 08:00:17,527] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:18,733] INFO : Step 77000/500000 rank0 | Train Loss: 4.4292 | Train Loss Unweighted: 2.5863 | Train Loss Components: Classification: 0.0222 | Classification_binary: 2.4156 | Regression_cos_phi: 0.0085 | Regression_energy: 0.0628 | Regression_eta: 0.0053 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2474 | Classification_binary: 2.4156 | Regression_cos_phi: 0.0942 | Regression_energy: 0.6691 | Regression_eta: 0.0427 | Regression_pt: 0.8874 | Regression_sin_phi: 0.0728 | LR: 2.70e-04 | DataLoad Time: 0.0004s | Model Forward Time: 0.5820s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0411 | max_loss: 0.0043 | max_backward: 0.0438 | max_optimizer_step: 0.0732 | max_tensorboard_logging: 0.1443 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2953.5 | uss_MB: 1886.2 | child_rss_MB: 22538.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 08:01:22,247] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 08:01:22,247] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:12,890] INFO : Step 78000/500000 rank0 | Train Loss: 5.2639 | Train Loss Unweighted: 2.8163 | Train Loss Components: Classification: 0.0229 | Classification_binary: 2.5899 | Regression_cos_phi: 0.0113 | Regression_energy: 0.0900 | Regression_eta: 0.0065 | Regression_pt: 0.0882 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2556 | Classification_binary: 2.5899 | Regression_cos_phi: 0.1248 | Regression_energy: 0.9591 | Regression_eta: 0.0525 | Regression_pt: 1.2012 | Regression_sin_phi: 0.0807 | LR: 2.75e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5110s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0028 | max_backward: 0.0394 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1307 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2953.5 | uss_MB: 1886.1 | child_rss_MB: 22541.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:02:12,892] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 08:02:12,962] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 08:02:26,268] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 08:02:26,268] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:03:29,920] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 08:03:29,920] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:04:06,490] INFO : Step 79000/500000 rank0 | Train Loss: 4.9621 | Train Loss Unweighted: 2.8435 | Train Loss Components: Classification: 0.0197 | Classification_binary: 2.6480 | Regression_cos_phi: 0.0034 | Regression_energy: 0.0823 | Regression_eta: 0.0060 | Regression_pt: 0.0791 | Regression_sin_phi: 0.0049 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2201 | Classification_binary: 2.6480 | Regression_cos_phi: 0.0380 | Regression_energy: 0.8769 | Regression_eta: 0.0482 | Regression_pt: 1.0778 | Regression_sin_phi: 0.0531 | LR: 2.80e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0378 | max_loss: 0.0032 | max_backward: 0.0413 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1209 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2953.2 | uss_MB: 1885.9 | child_rss_MB: 22541.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 08:04:33,828] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 08:04:33,828] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:37,421] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 08:05:37,421] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:59,933] INFO : Step 80000/500000 rank0 | Train Loss: 4.6948 | Train Loss Unweighted: 2.6752 | Train Loss Components: Classification: 0.0223 | Classification_binary: 2.4868 | Regression_cos_phi: 0.0065 | Regression_energy: 0.0717 | Regression_eta: 0.0092 | Regression_pt: 0.0706 | Regression_sin_phi: 0.0081 | Task Weights: Classification: 11.1592 | Classification_binary: 1.0000 | Regression_cos_phi: 11.0399 | Regression_energy: 10.6547 | Regression_eta: 8.0492 | Regression_pt: 13.6269 | Regression_sin_phi: 10.8062 | Task Weighted Losses: Classification: 0.2489 | Classification_binary: 2.4868 | Regression_cos_phi: 0.0718 | Regression_energy: 0.7637 | Regression_eta: 0.0742 | Regression_pt: 0.9615 | Regression_sin_phi: 0.0880 | LR: 2.85e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4901s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0395 | max_optimizer_step: 0.0642 | max_tensorboard_logging: 0.1237 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2953.3 | uss_MB: 1886.0 | child_rss_MB: 22542.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:05:59,935] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 08:06:00,003] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 08:06:00,004] INFO : Running validation on rank0 +rank0 [2026-11-14 08:06:00,004] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 08:06:00,004] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 08:06:00,158] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 08:06:00,282] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 08:06:00,403] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 08:06:00,474] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 08:06:00,544] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 08:06:00,612] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 08:06:00,681] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 08:06:00,749] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 08:06:00,852] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 08:06:00,972] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 08:06:01,091] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 08:06:01,159] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 08:06:01,227] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 08:06:01,294] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 08:06:01,361] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 08:06:01,429] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 08:06:01,537] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 08:06:01,544] INFO : plotted pt_target histogram +rank0 [2026-11-14 08:06:01,547] INFO : plotted pt_pred histogram +rank0 [2026-11-14 08:06:01,549] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 08:06:01,550] INFO : plotted eta_target histogram +rank0 [2026-11-14 08:06:01,553] INFO : plotted eta_pred histogram +rank0 [2026-11-14 08:06:01,555] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 08:06:01,557] INFO : plotted sphi_target histogram +rank0 [2026-11-14 08:06:01,559] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 08:06:01,562] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 08:06:01,564] INFO : plotted cphi_target histogram +rank0 [2026-11-14 08:06:01,566] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 08:06:01,568] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 08:06:01,570] INFO : plotted energy_target histogram +rank0 [2026-11-14 08:06:01,572] INFO : plotted energy_pred histogram +rank0 [2026-11-14 08:06:01,575] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 08:06:03,101] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.6948 | Valid Loss=10.1646 | Train Loss Unweighted=2.6752 | Valid Loss Unweighted=4.3723 | Stale=3 | ETA=799.4m +rank0 [2026-11-14 08:06:03,101] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 08:06:03,512] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 08:06:03,519] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 08:06:03,562] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 08:06:04,055] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 08:06:04,309] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 08:06:04,558] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 08:06:04,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 08:06:05,082] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 08:06:05,345] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 08:06:05,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 08:06:05,860] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 08:06:06,120] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 08:06:06,379] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 08:06:06,636] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 08:06:06,903] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 08:06:07,160] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 08:06:07,419] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 08:06:07,682] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 08:06:07,916] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 08:06:07,981] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 08:06:07,988] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:06:08,577] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_20mm_seed12345_20261114_053319_006812/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:06:08,973] INFO : Plotted number of elements +rank0 [2026-11-14 08:06:10,346] INFO : Plotted jets +rank0 [2026-11-14 08:06:11,361] INFO : Plotted jet ratio +rank0 [2026-11-14 08:06:11,813] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 08:06:11,813] INFO : Training completed. Total time on device 0: 152.409min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..aa77973873c82f3ef8536d8c2c93140dadc47298 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:597841121f1607f15c17a060070ead10bcca5d604aa3b89947407c4382034f34 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-80000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-80000.pth new file mode 100644 index 0000000000000000000000000000000000000000..aa85920ec45e6424ec4576b172640819ca35bedd --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-80000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbe6626c3a111c6e4e0d2810a1e7444c2e1e0aa6177c6d74d1376d47d40f95d4 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..0618219b7643e99f8d2f89e25c386ef13e5f4576 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 3.8801279067993164, "Classification": 0.03810478001832962, "Regression_pt": 0.212041974067688, "Regression_eta": 0.0161382295191288, "Regression_sin_phi": 0.010906083509325981, "Regression_cos_phi": 0.010633113794028759, "Regression_energy": 0.20640519261360168, "Total": 9.628539085388184, "Total_unweighted": 4.374357280321419}, "train_group": {}, "valid": {"Classification_binary": 3.9199724197387695, "Classification": 0.036035362631082535, "Regression_pt": 0.1955484002828598, "Regression_eta": 0.01328732818365097, "Regression_sin_phi": 0.011556842364370823, "Regression_cos_phi": 0.010772744193673134, "Regression_energy": 0.1927991360425949, "Total": 9.27129077911377, "Total_unweighted": 4.379972233437002, "metrics/particle/class_1/efficiency": 0.8897920846939087, "metrics/particle/class_1/pid_accuracy": 0.9817290306091309, "metrics/particle/class_2/efficiency": 0.19014890491962433, "metrics/particle/class_2/pid_accuracy": 0.3012048304080963, "metrics/particle/class_3/efficiency": 0.6979695558547974, "metrics/particle/class_3/pid_accuracy": 0.9765656590461731, "metrics/particle/class_4/efficiency": 0.8644067645072937, "metrics/particle/class_4/pid_accuracy": 0.03529411926865578, "metrics/particle/class_5/efficiency": 0.9340659379959106, "metrics/particle/class_5/pid_accuracy": 0.10000000149011612, "metrics/particle/count/bias_mean": -0.7599999904632568, "metrics/particle/count/mae": 5.223999977111816, "metrics/particle/count/prediction_mean": 97.43199920654297, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.080165795981884, "metrics/particle/event/energy_response_mean": 0.9716163873672485, "metrics/particle/event/met_abs_error": 14.122517585754395, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07791351526975632, "metrics/particle/event/scalar_pt_response_mean": 0.9780303835868835, "metrics/particle/event/vector_pt_closure": 0.08969078212976456, "metrics/particle/matched/delta_eta_abs_mean": 0.02128848060965538, "metrics/particle/matched/delta_phi_abs_mean": 0.01923837512731552, "metrics/particle/matched/delta_r_mean": 0.03195858374238014, "metrics/particle/matched/energy_relative_abs_mean": 0.09959471970796585, "metrics/particle/matched/pid_accuracy": 0.9464506506919861, "metrics/particle/matched/pt_relative_abs_mean": 0.09910009801387787, "metrics/particle/matching/duplicate_fraction": 0.04232695698738098, "metrics/particle/matching/efficiency": 0.7569252252578735, "metrics/particle/matching/f1": 0.7598658800125122, "metrics/particle/matching/purity": 0.7628294825553894, "metrics/particle/matching/target_coverage_dr0p05": 0.6315789222717285, "metrics/particle/matching/target_coverage_dr0p10": 0.7914697527885437, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6696268320083618, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8147303462028503}, "step_train_time": 0.5782678127288818, "step_valid_time": 2.8788962370017543, "step_total_time": 1126.3043642044067, "elapsed_training_loop_seconds": 1126.3043642044067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.992089901948662, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2177824354598168, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1512, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1315, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1232, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8749168330006654, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8697089947089947, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8723051409618574, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1302910052910053, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8196939454424484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8148148148148148, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8172470978441128, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8749168330006654, "step_test_time": 5.44747846503742, "step_plot_time": 4.138989772996865, "step_validation_cycle_time": 12.979508138028905} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..8bc30277bcbdc353376507c30adcbdc4e7995375 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 3.69901180267334, "Classification": 0.04142634570598602, "Regression_pt": 0.19263844192028046, "Regression_eta": 0.00833602249622345, "Regression_sin_phi": 0.01051898580044508, "Regression_cos_phi": 0.01037190854549408, "Regression_energy": 0.18399272859096527, "Total": 8.925642013549805, "Total_unweighted": 4.146296235732734}, "train_group": {}, "valid": {"Classification_binary": 3.7766225337982178, "Classification": 0.03446405380964279, "Regression_pt": 0.18786317110061646, "Regression_eta": 0.012148672714829445, "Regression_sin_phi": 0.01067233644425869, "Regression_cos_phi": 0.009886207990348339, "Regression_energy": 0.18585623800754547, "Total": 8.907792091369629, "Total_unweighted": 4.217513213865459, "metrics/particle/class_1/efficiency": 0.8885086178779602, "metrics/particle/class_1/pid_accuracy": 0.983147144317627, "metrics/particle/class_2/efficiency": 0.21935853362083435, "metrics/particle/class_2/pid_accuracy": 0.3603133261203766, "metrics/particle/class_3/efficiency": 0.7277683615684509, "metrics/particle/class_3/pid_accuracy": 0.9834668040275574, "metrics/particle/class_4/efficiency": 0.8745762705802917, "metrics/particle/class_4/pid_accuracy": 0.06976744532585144, "metrics/particle/class_5/efficiency": 0.9560439586639404, "metrics/particle/class_5/pid_accuracy": 0.17241379618644714, "metrics/particle/count/bias_mean": 1.6360000371932983, "metrics/particle/count/mae": 4.892000198364258, "metrics/particle/count/prediction_mean": 99.8280029296875, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.06250141561031342, "metrics/particle/event/energy_response_mean": 1.0056673288345337, "metrics/particle/event/met_abs_error": 12.008092880249023, "metrics/particle/event/scalar_pt_relative_abs_error": 0.066441610455513, "metrics/particle/event/scalar_pt_response_mean": 1.013304591178894, "metrics/particle/event/vector_pt_closure": 0.07839293777942657, "metrics/particle/matched/delta_eta_abs_mean": 0.019044453278183937, "metrics/particle/matched/delta_phi_abs_mean": 0.018297363072633743, "metrics/particle/matched/delta_r_mean": 0.02938239462673664, "metrics/particle/matched/energy_relative_abs_mean": 0.09846734255552292, "metrics/particle/matched/pid_accuracy": 0.9507945775985718, "metrics/particle/matched/pt_relative_abs_mean": 0.09911172837018967, "metrics/particle/matching/duplicate_fraction": 0.041311055421829224, "metrics/particle/matching/efficiency": 0.7715903520584106, "metrics/particle/matching/f1": 0.7652156352996826, "metrics/particle/matching/purity": 0.7589454054832458, "metrics/particle/matching/target_coverage_dr0p05": 0.668404757976532, "metrics/particle/matching/target_coverage_dr0p10": 0.8106159567832947, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6671012043952942, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8271957039833069}, "step_train_time": 0.6136739253997803, "step_valid_time": 2.993058067979291, "step_total_time": 2265.9464337825775, "elapsed_training_loop_seconds": 2265.9464337825775, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0204826240354712, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21206622614348425, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1518, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1342, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1267, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8928809048569527, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8840579710144928, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8884475339291625, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11594202898550725, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8429807052561543, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8346508563899868, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.838795100959947, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8928809048569527, "step_test_time": 5.079598925076425, "step_plot_time": 3.317665445036255, "step_validation_cycle_time": 11.932410289999098} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..6761c81dc9a0b71a69a4ad8901c1ce88e35bb2dc --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 3.611459255218506, "Classification": 0.03267139568924904, "Regression_pt": 0.18303164839744568, "Regression_eta": 0.009659914299845695, "Regression_sin_phi": 0.00777897983789444, "Regression_cos_phi": 0.008208264596760273, "Regression_energy": 0.18026375770568848, "Total": 8.530559539794922, "Total_unweighted": 4.0330732157453895}, "train_group": {}, "valid": {"Classification_binary": 3.645770311355591, "Classification": 0.033657003194093704, "Regression_pt": 0.18110030889511108, "Regression_eta": 0.011777933686971664, "Regression_sin_phi": 0.010274147614836693, "Regression_cos_phi": 0.009440996684134007, "Regression_energy": 0.1801876574754715, "Total": 8.607107162475586, "Total_unweighted": 4.0722083589062095, "metrics/particle/class_1/efficiency": 0.9046804308891296, "metrics/particle/class_1/pid_accuracy": 0.9835429787635803, "metrics/particle/class_2/efficiency": 0.23253150284290314, "metrics/particle/class_2/pid_accuracy": 0.4211822748184204, "metrics/particle/class_3/efficiency": 0.7491070032119751, "metrics/particle/class_3/pid_accuracy": 0.9821809530258179, "metrics/particle/class_4/efficiency": 0.8677965998649597, "metrics/particle/class_4/pid_accuracy": 0.0859375, "metrics/particle/class_5/efficiency": 0.9505494236946106, "metrics/particle/class_5/pid_accuracy": 0.23699422180652618, "metrics/particle/count/bias_mean": 1.4479999542236328, "metrics/particle/count/mae": 4.71999979019165, "metrics/particle/count/prediction_mean": 99.63999938964844, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.05865353345870972, "metrics/particle/event/energy_response_mean": 0.9696914553642273, "metrics/particle/event/met_abs_error": 10.209823608398438, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06025730073451996, "metrics/particle/event/scalar_pt_response_mean": 0.9686961770057678, "metrics/particle/event/vector_pt_closure": 0.06612108647823334, "metrics/particle/matched/delta_eta_abs_mean": 0.019225535914301872, "metrics/particle/matched/delta_phi_abs_mean": 0.017560824751853943, "metrics/particle/matched/delta_r_mean": 0.02904057316482067, "metrics/particle/matched/energy_relative_abs_mean": 0.09288863092660904, "metrics/particle/matched/pid_accuracy": 0.9526758790016174, "metrics/particle/matched/pt_relative_abs_mean": 0.08938843011856079, "metrics/particle/matching/duplicate_fraction": 0.04058611020445824, "metrics/particle/matching/efficiency": 0.7893514633178711, "metrics/particle/matching/f1": 0.7835739254951477, "metrics/particle/matching/purity": 0.7778803706169128, "metrics/particle/matching/target_coverage_dr0p05": 0.6780593395233154, "metrics/particle/matching/target_coverage_dr0p10": 0.8182743787765503, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6978165507316589, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8449975848197937}, "step_train_time": 0.6357507705688477, "step_valid_time": 3.206355314934626, "step_total_time": 3400.364832878113, "elapsed_training_loop_seconds": 3400.364832878113, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9928260894621828, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17460787176061465, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1495, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1334, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1294, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8875582168995343, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8923076923076924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8899266177451635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1076923076923077, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8609447771124418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8655518394648829, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8632421614409607, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8875582168995343, "step_test_time": 5.20166156894993, "step_plot_time": 3.600243377033621, "step_validation_cycle_time": 12.625663078040816} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..34436367bd9859504a9d6c6b709a1b889d6e1e21 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 3.6322875022888184, "Classification": 0.027793845161795616, "Regression_pt": 0.21417468786239624, "Regression_eta": 0.00600126339122653, "Regression_sin_phi": 0.007698698900640011, "Regression_cos_phi": 0.0044946870766580105, "Regression_energy": 0.21077248454093933, "Total": 9.155501365661621, "Total_unweighted": 4.103223169222474}, "train_group": {}, "valid": {"Classification_binary": 3.4720234870910645, "Classification": 0.030423684045672417, "Regression_pt": 0.17723196744918823, "Regression_eta": 0.011473765596747398, "Regression_sin_phi": 0.010095344856381416, "Regression_cos_phi": 0.009120364673435688, "Regression_energy": 0.17721697688102722, "Total": 8.307068824768066, "Total_unweighted": 3.887585590593517, "metrics/particle/class_1/efficiency": 0.9098142981529236, "metrics/particle/class_1/pid_accuracy": 0.9806263446807861, "metrics/particle/class_2/efficiency": 0.23081329464912415, "metrics/particle/class_2/pid_accuracy": 0.49875929951667786, "metrics/particle/class_3/efficiency": 0.7360405921936035, "metrics/particle/class_3/pid_accuracy": 0.9779055118560791, "metrics/particle/class_4/efficiency": 0.8915254473686218, "metrics/particle/class_4/pid_accuracy": 0.1596958190202713, "metrics/particle/class_5/efficiency": 0.9670329689979553, "metrics/particle/class_5/pid_accuracy": 0.34659090638160706, "metrics/particle/count/bias_mean": 0.06400000303983688, "metrics/particle/count/mae": 4.480000019073486, "metrics/particle/count/prediction_mean": 98.25599670410156, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.06618068367242813, "metrics/particle/event/energy_response_mean": 0.9489498138427734, "metrics/particle/event/met_abs_error": 9.628042221069336, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06637825816869736, "metrics/particle/event/scalar_pt_response_mean": 0.9510255455970764, "metrics/particle/event/vector_pt_closure": 0.060736238956451416, "metrics/particle/matched/delta_eta_abs_mean": 0.01966560073196888, "metrics/particle/matched/delta_phi_abs_mean": 0.018964363262057304, "metrics/particle/matched/delta_r_mean": 0.030434999614953995, "metrics/particle/matched/energy_relative_abs_mean": 0.0939435362815857, "metrics/particle/matched/pid_accuracy": 0.9524993300437927, "metrics/particle/matched/pt_relative_abs_mean": 0.08853597939014435, "metrics/particle/matching/duplicate_fraction": 0.03606904298067093, "metrics/particle/matching/efficiency": 0.7864184379577637, "metrics/particle/matching/f1": 0.786162257194519, "metrics/particle/matching/purity": 0.785906195640564, "metrics/particle/matching/target_coverage_dr0p05": 0.6590353846549988, "metrics/particle/matching/target_coverage_dr0p10": 0.8137526512145996, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7030715346336365, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8446309566497803}, "step_train_time": 0.6099364757537842, "step_valid_time": 3.170292533002794, "step_total_time": 4538.984757184982, "elapsed_training_loop_seconds": 4538.984757184982, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9726282734925313, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1613982019848187, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1350, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1318, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8982035928143712, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9090909090909091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9036144578313253, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09090909090909091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8769128409846972, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8875420875420875, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8821954484605087, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8982035928143712, "step_test_time": 5.0010439089965075, "step_plot_time": 3.3863687029806897, "step_validation_cycle_time": 12.09868680499494} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..7e6c1b5ba6b4aedfb16f72d5508e0d7480fe831c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 3.1958231925964355, "Classification": 0.027645856142044067, "Regression_pt": 0.11294733732938766, "Regression_eta": 0.006810533814132214, "Regression_sin_phi": 0.005862528458237648, "Regression_cos_phi": 0.005946355406194925, "Regression_energy": 0.11807279288768768, "Total": 6.4145097732543945, "Total_unweighted": 3.4731085966341197}, "train_group": {}, "valid": {"Classification_binary": 3.381571054458618, "Classification": 0.028732560575008392, "Regression_pt": 0.17614305019378662, "Regression_eta": 0.011399726383388042, "Regression_sin_phi": 0.009506650269031525, "Regression_cos_phi": 0.00857677310705185, "Regression_energy": 0.17499181628227234, "Total": 8.14702033996582, "Total_unweighted": 3.790921631269157, "metrics/particle/class_1/efficiency": 0.9075896143913269, "metrics/particle/class_1/pid_accuracy": 0.9815216660499573, "metrics/particle/class_2/efficiency": 0.26575028896331787, "metrics/particle/class_2/pid_accuracy": 0.5431034564971924, "metrics/particle/class_3/efficiency": 0.749576985836029, "metrics/particle/class_3/pid_accuracy": 0.9803110361099243, "metrics/particle/class_4/efficiency": 0.8949152827262878, "metrics/particle/class_4/pid_accuracy": 0.20075757801532745, "metrics/particle/class_5/efficiency": 0.9505494236946106, "metrics/particle/class_5/pid_accuracy": 0.46820810437202454, "metrics/particle/count/bias_mean": 1.4279999732971191, "metrics/particle/count/mae": 4.708000183105469, "metrics/particle/count/prediction_mean": 99.62000274658203, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.05575883761048317, "metrics/particle/event/energy_response_mean": 1.0051525831222534, "metrics/particle/event/met_abs_error": 9.073647499084473, "metrics/particle/event/scalar_pt_relative_abs_error": 0.053215980529785156, "metrics/particle/event/scalar_pt_response_mean": 1.0004549026489258, "metrics/particle/event/vector_pt_closure": 0.057693321257829666, "metrics/particle/matched/delta_eta_abs_mean": 0.020631790161132812, "metrics/particle/matched/delta_phi_abs_mean": 0.01718575693666935, "metrics/particle/matched/delta_r_mean": 0.02991732954978943, "metrics/particle/matched/energy_relative_abs_mean": 0.10396204888820648, "metrics/particle/matched/pid_accuracy": 0.9554460644721985, "metrics/particle/matched/pt_relative_abs_mean": 0.09626244008541107, "metrics/particle/matching/duplicate_fraction": 0.03846617043018341, "metrics/particle/matching/efficiency": 0.7936288118362427, "metrics/particle/matching/f1": 0.7878996133804321, "metrics/particle/matching/purity": 0.7822525501251221, "metrics/particle/matching/target_coverage_dr0p05": 0.6710119247436523, "metrics/particle/matching/target_coverage_dr0p10": 0.827521562576294, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.693416953086853, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.846097469329834}, "step_train_time": 0.647233247756958, "step_valid_time": 3.054671567049809, "step_total_time": 5676.757130146027, "elapsed_training_loop_seconds": 5676.757130146027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.01013090238696, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1482324458000036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1344, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1320, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8942115768463074, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8983957219251337, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8962987662554185, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10160427807486631, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8782435129740519, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8823529411764706, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8802934311437146, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8942115768463074, "step_test_time": 5.170041171950288, "step_plot_time": 3.3884416540386155, "step_validation_cycle_time": 12.137988526024856} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_60000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_60000.json new file mode 100644 index 0000000000000000000000000000000000000000..4923104989150b9587e0d23b1c5fc5fe1479196a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_60000.json @@ -0,0 +1 @@ +{"epoch": 106.76156583629893, "train": {"Classification_binary": 3.1039798259735107, "Classification": 0.02212143875658512, "Regression_pt": 0.12423115968704224, "Regression_eta": 0.00576411560177803, "Regression_sin_phi": 0.005648416467010975, "Regression_cos_phi": 0.008299403823912144, "Regression_energy": 0.12189704179763794, "Total": 6.465737342834473, "Total_unweighted": 3.391941402107477}, "train_group": {}, "valid": {"Classification_binary": 3.382138967514038, "Classification": 0.028831934556365013, "Regression_pt": 0.18730062246322632, "Regression_eta": 0.011242445558309555, "Regression_sin_phi": 0.0094186095520854, "Regression_cos_phi": 0.008419165387749672, "Regression_energy": 0.18714621663093567, "Total": 8.419062614440918, "Total_unweighted": 3.8144979616627097, "metrics/particle/class_1/efficiency": 0.910498857498169, "metrics/particle/class_1/pid_accuracy": 0.9828963279724121, "metrics/particle/class_2/efficiency": 0.274341344833374, "metrics/particle/class_2/pid_accuracy": 0.5407097935676575, "metrics/particle/class_3/efficiency": 0.7360405921936035, "metrics/particle/class_3/pid_accuracy": 0.9804597496986389, "metrics/particle/class_4/efficiency": 0.8779661059379578, "metrics/particle/class_4/pid_accuracy": 0.2046332061290741, "metrics/particle/class_5/efficiency": 0.9505494236946106, "metrics/particle/class_5/pid_accuracy": 0.47398844361305237, "metrics/particle/count/bias_mean": 1.1360000371932983, "metrics/particle/count/mae": 4.952000141143799, "metrics/particle/count/prediction_mean": 99.3280029296875, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.05775557458400726, "metrics/particle/event/energy_response_mean": 0.967547595500946, "metrics/particle/event/met_abs_error": 9.057416915893555, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05660390481352806, "metrics/particle/event/scalar_pt_response_mean": 0.9691387414932251, "metrics/particle/event/vector_pt_closure": 0.057609789073467255, "metrics/particle/matched/delta_eta_abs_mean": 0.020360026508569717, "metrics/particle/matched/delta_phi_abs_mean": 0.018209772184491158, "metrics/particle/matched/delta_r_mean": 0.030509361997246742, "metrics/particle/matched/energy_relative_abs_mean": 0.09535717219114304, "metrics/particle/matched/pid_accuracy": 0.956041693687439, "metrics/particle/matched/pt_relative_abs_mean": 0.0889296606183052, "metrics/particle/matching/duplicate_fraction": 0.037975192070007324, "metrics/particle/matching/efficiency": 0.7895551323890686, "metrics/particle/matching/f1": 0.7850141525268555, "metrics/particle/matching/purity": 0.7805251479148865, "metrics/particle/matching/target_coverage_dr0p05": 0.6642088890075684, "metrics/particle/matching/target_coverage_dr0p10": 0.8190076351165771, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7015643119812012, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8464640974998474}, "step_train_time": 0.6053304672241211, "step_valid_time": 3.093076309072785, "step_total_time": 6818.281809091568, "elapsed_training_loop_seconds": 6818.281809091568, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9893103211290541, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1474922705564209, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1495, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1365, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1347, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.908183632734531, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9130434782608695, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9106070713809206, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08695652173913043, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8962075848303394, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9010033444816054, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8985990660440294, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.908183632734531, "step_test_time": 5.295604826998897, "step_plot_time": 3.4183156420476735, "step_validation_cycle_time": 12.25693263602443} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_70000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_70000.json new file mode 100644 index 0000000000000000000000000000000000000000..20feeb00eb774bf97f99ad65af44aaef6f9fec0c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_70000.json @@ -0,0 +1 @@ +{"epoch": 124.55516014234875, "train": {"Classification_binary": 2.558375835418701, "Classification": 0.022474486380815506, "Regression_pt": 0.08102043718099594, "Regression_eta": 0.004143773578107357, "Regression_sin_phi": 0.006039157975465059, "Regression_cos_phi": 0.004168045707046986, "Regression_energy": 0.08022896945476532, "Total": 4.863192081451416, "Total_unweighted": 2.7564507056958973}, "train_group": {}, "valid": {"Classification_binary": 3.4844369888305664, "Classification": 0.029234251007437706, "Regression_pt": 0.19719764590263367, "Regression_eta": 0.011160418391227722, "Regression_sin_phi": 0.009184425696730614, "Regression_cos_phi": 0.00821200292557478, "Regression_energy": 0.19712796807289124, "Total": 8.755358695983887, "Total_unweighted": 3.936553700827062, "metrics/particle/class_1/efficiency": 0.9147771000862122, "metrics/particle/class_1/pid_accuracy": 0.9782994985580444, "metrics/particle/class_2/efficiency": 0.28293243050575256, "metrics/particle/class_2/pid_accuracy": 0.6093117594718933, "metrics/particle/class_3/efficiency": 0.7368866205215454, "metrics/particle/class_3/pid_accuracy": 0.9775481820106506, "metrics/particle/class_4/efficiency": 0.8779661059379578, "metrics/particle/class_4/pid_accuracy": 0.2277992218732834, "metrics/particle/class_5/efficiency": 0.9560439586639404, "metrics/particle/class_5/pid_accuracy": 0.5114942789077759, "metrics/particle/count/bias_mean": 1.1720000505447388, "metrics/particle/count/mae": 5.164000034332275, "metrics/particle/count/prediction_mean": 99.36399841308594, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.0582573302090168, "metrics/particle/event/energy_response_mean": 0.99004727602005, "metrics/particle/event/met_abs_error": 10.188994407653809, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0562204048037529, "metrics/particle/event/scalar_pt_response_mean": 0.9905123710632324, "metrics/particle/event/vector_pt_closure": 0.06067505106329918, "metrics/particle/matched/delta_eta_abs_mean": 0.020473554730415344, "metrics/particle/matched/delta_phi_abs_mean": 0.017259307205677032, "metrics/particle/matched/delta_r_mean": 0.02985033579170704, "metrics/particle/matched/energy_relative_abs_mean": 0.09614916145801544, "metrics/particle/matched/pid_accuracy": 0.9544636607170105, "metrics/particle/matched/pt_relative_abs_mean": 0.08968900144100189, "metrics/particle/matching/duplicate_fraction": 0.03884706646203995, "metrics/particle/matching/efficiency": 0.7926104068756104, "metrics/particle/matching/f1": 0.7879082560539246, "metrics/particle/matching/purity": 0.7832615375518799, "metrics/particle/matching/target_coverage_dr0p05": 0.6671419143676758, "metrics/particle/matching/target_coverage_dr0p10": 0.8229998350143433, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7061675190925598, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8471565842628479}, "step_train_time": 0.6117126941680908, "step_valid_time": 3.107459489023313, "step_total_time": 7960.553455352783, "elapsed_training_loop_seconds": 7960.553455352783, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028127180795652, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14930362396722285, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1366, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1343, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9088489687292083, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9094540612516645, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9091514143094842, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09054593874833555, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.89354624085163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8941411451398136, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8938435940099834, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9088489687292083, "step_test_time": 5.032289869035594, "step_plot_time": 3.395246233907528, "step_validation_cycle_time": 11.9655657740077} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_80000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_80000.json new file mode 100644 index 0000000000000000000000000000000000000000..5adbfb9680610fcf5d1d147c5afb1a56012c80c7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/history/step_80000.json @@ -0,0 +1 @@ +{"epoch": 142.34875444839858, "train": {"Classification_binary": 2.4390575885772705, "Classification": 0.022052207961678505, "Regression_pt": 0.06820543855428696, "Regression_eta": 0.00441073440015316, "Regression_sin_phi": 0.00659380154684186, "Regression_cos_phi": 0.006270726677030325, "Regression_energy": 0.0700419470667839, "Total": 4.495077610015869, "Total_unweighted": 2.616632444784045}, "train_group": {}, "valid": {"Classification_binary": 3.897092819213867, "Classification": 0.030965639278292656, "Regression_pt": 0.21643081307411194, "Regression_eta": 0.01120127271860838, "Regression_sin_phi": 0.009149590507149696, "Regression_cos_phi": 0.008228825405240059, "Regression_energy": 0.21636423468589783, "Total": 9.64253044128418, "Total_unweighted": 4.389433194883168, "metrics/particle/class_1/efficiency": 0.9118679165840149, "metrics/particle/class_1/pid_accuracy": 0.9756028652191162, "metrics/particle/class_2/efficiency": 0.30584192276000977, "metrics/particle/class_2/pid_accuracy": 0.5973782539367676, "metrics/particle/class_3/efficiency": 0.7187440991401672, "metrics/particle/class_3/pid_accuracy": 0.9716191291809082, "metrics/particle/class_4/efficiency": 0.8711864352226257, "metrics/particle/class_4/pid_accuracy": 0.22568093240261078, "metrics/particle/class_5/efficiency": 0.9615384340286255, "metrics/particle/class_5/pid_accuracy": 0.4571428596973419, "metrics/particle/count/bias_mean": 1.2640000581741333, "metrics/particle/count/mae": 5.176000118255615, "metrics/particle/count/prediction_mean": 99.45600128173828, "metrics/particle/count/target_mean": 98.19200134277344, "metrics/particle/event/energy_relative_abs_error": 0.055662352591753006, "metrics/particle/event/energy_response_mean": 1.0018644332885742, "metrics/particle/event/met_abs_error": 9.754003524780273, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05415999889373779, "metrics/particle/event/scalar_pt_response_mean": 0.9992068409919739, "metrics/particle/event/vector_pt_closure": 0.06219591945409775, "metrics/particle/matched/delta_eta_abs_mean": 0.01929353177547455, "metrics/particle/matched/delta_phi_abs_mean": 0.017592696473002434, "metrics/particle/matched/delta_r_mean": 0.029092052951455116, "metrics/particle/matched/energy_relative_abs_mean": 0.10002852231264114, "metrics/particle/matched/pid_accuracy": 0.948829710483551, "metrics/particle/matched/pt_relative_abs_mean": 0.09092997759580612, "metrics/particle/matching/duplicate_fraction": 0.04033944755792618, "metrics/particle/matching/efficiency": 0.78495192527771, "metrics/particle/matching/f1": 0.7799320220947266, "metrics/particle/matching/purity": 0.7749758958816528, "metrics/particle/matching/target_coverage_dr0p05": 0.6568763256072998, "metrics/particle/matching/target_coverage_dr0p10": 0.8143637180328369, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6994459629058838, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8436939716339111}, "step_train_time": 0.6702625751495361, "step_valid_time": 3.076894370955415, "step_total_time": 9098.917045593262, "elapsed_training_loop_seconds": 9098.917045593262, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0144897357119833, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16302165767472698, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1520, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1369, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1345, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9108449767132402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9006578947368421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9057227919285478, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0993421052631579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8948769128409847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8848684210526315, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8898445253059875, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9108449767132402, "step_test_time": 5.026243382017128, "step_plot_time": 3.320617015939206, "step_validation_cycle_time": 11.905219406937249} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..530d763d7f06e3277cb75e3835c5989df8f58704 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_25mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..2812bc2a09daa47f18bf155a92f7a78fb1468db2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed54e7f349afc21513d72c44cad22893c34c04c02a95fda1b90fc3ab04125d96 +size 4394 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..24f79adaca06182f8838d653d81779c729702b00 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..6c0991b2d50cd62782ddd0560f15bda733fb89c9 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b9a53e18ffb69486f6b471ecb7cc178e4d9f4416 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..5cea8b849cb14f7dae13e3a6e07bc3372d09eed3 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..03a79597545e8cbbcf7b8007ce6973dde129ca0c Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..d8e6af25035aefe22293f4d39a1e6a50473907ec Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2979eca02c8c7fb4c53dff081c2a9cdf414838cd Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..c8bf6a1dcdc25ec8fd16286c6ae8e6879e89d4b3 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ea885bae6127f7d6e8d32d78d918df3132bcdc0a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..d20f2fa9b2a846d944eb5b963f57efabb67a1081 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9171f0da3d2896e89cf2503fc04380224974fd7e Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..b1b913e8c4df96587e4bdb1de5afd4b9c3268d5a Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/plots_step_80000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/runs/train/events.out.tfevents.1794652450.workergpu159.3306172.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/runs/train/events.out.tfevents.1794652450.workergpu159.3306172.0 new file mode 100644 index 0000000000000000000000000000000000000000..0ad64b695ce17794b44d86e1aefcec33c6b32d3d --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/runs/train/events.out.tfevents.1794652450.workergpu159.3306172.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b65f183ca4e3fa279b6c26a200dc92c4cfd0d97755ab2d3df8a6cb4088a0b1a3 +size 2953648 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/runs/valid/events.out.tfevents.1794652450.workergpu159.3306172.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/runs/valid/events.out.tfevents.1794652450.workergpu159.3306172.1 new file mode 100644 index 0000000000000000000000000000000000000000..e8693270bdc5e3721593a93b7ab5daefc6f5503b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/runs/valid/events.out.tfevents.1794652450.workergpu159.3306172.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77cd006dd27347c93f69b686db0f647652a9237fd185a94cad1be16d1225a76e +size 4013591 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..03ce29adb46a8af8744d62f5e57c8548eaabd8de --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_hcal_25mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_25mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_25mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_25mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bebd471ea3bda6b1cc1c676f9e26137cc12b13c4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_25mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/train.log new file mode 100644 index 0000000000000000000000000000000000000000..2373aff9a8f7e58eaedc8456ff88156e3bfd2dda --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/train.log @@ -0,0 +1,1353 @@ +rank0 [2026-11-14 05:34:06,234] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-14 05:34:06,234] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 05:34:06,956] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 05:34:06,956] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 05:34:06,958] INFO : Instantiating model +rank0 [2026-11-14 05:34:06,980] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 05:34:06,980] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 05:34:06,982] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 05:34:06,982] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 05:34:06,982] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 05:34:06,982] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 05:34:06,982] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,985] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 05:34:06,985] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,988] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 05:34:06,988] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,990] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 05:34:06,990] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,993] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 05:34:06,993] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,996] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 05:34:06,996] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,999] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 05:34:06,999] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 05:34:06,999] INFO : Initializing output DNNs +rank0 [2026-11-14 05:34:06,999] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:07,003] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:07,010] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 05:34:07,010] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 05:34:07,010] INFO : jagged_attention enabled=True +rank0 [2026-11-14 05:34:07,010] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 05:34:07,010] INFO : task_queries enabled=True +rank0 [2026-11-14 05:34:07,010] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 05:34:07,010] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 05:34:07,010] INFO : classification_query parameters: 256 +rank0 [2026-11-14 05:34:07,010] INFO : regression_query parameters: 256 +rank0 [2026-11-14 05:34:07,010] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 05:34:07,010] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 05:34:07,010] INFO : output_mode=elementwise +rank0 [2026-11-14 05:34:07,010] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 05:34:07,010] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 05:34:07,010] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 05:34:07,011] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 05:34:07,011] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 05:34:07,011] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 05:34:07,011] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 05:34:07,011] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 05:34:07,012] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 05:34:07,012] INFO : MLPF __init__ done +rank0 [2026-11-14 05:34:07,012] INFO : Instantiated model +rank0 [2026-11-14 05:34:07,013] INFO : Moving model to device rank=0 +rank0 [2026-11-14 05:34:07,021] INFO : Moved model to device rank=0 +rank0 [2026-11-14 05:34:07,022] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 05:34:07,125] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 05:34:07,125] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 05:34:07,126] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 05:34:07,126] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 05:34:07,126] INFO : Total parameters: 3910930 +rank0 [2026-11-14 05:34:07,128] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 05:34:07,133] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046 +rank0 [2026-11-14 05:34:07,133] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046 +rank0 [2026-11-14 05:34:07,133] INFO : Getting dataloaders +rank0 [2026-11-14 05:34:07,133] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:34:09,414] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,503] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,554] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,596] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,658] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,706] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,750] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,823] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,868] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,906] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,906] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 05:34:09,906] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 05:34:09,906] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 05:34:09,906] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:34:09,914] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,923] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,936] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,946] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,965] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,974] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,993] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:10,005] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:10,018] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:10,027] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:10,027] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 05:34:10,027] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 05:34:10,075] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:34:10,234] INFO : Got dataloaders +rank0 [2026-11-14 05:34:10,234] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 05:34:10,235] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 05:34:10,235] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 05:34:10,241] INFO : Creating train_iterator +rank0 [2026-11-14 05:34:10,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:34:10,270] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:34:10,431] INFO : Created train_iterator +rank0 [2026-11-14 05:34:23,321] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.080610990524292, 'Classification': 0.6321391302347183, 'Regression_pt': 0.5351871705055237, 'Regression_eta': 0.8981421908736229, 'Regression_sin_phi': 0.653542291522026, 'Regression_cos_phi': 0.6319048485159874, 'Regression_energy': 0.6792134913802147}, weights={'Classification_binary': 1.0, 'Classification': 11.201032638549805, 'Regression_pt': 13.230157852172852, 'Regression_eta': 7.88361930847168, 'Regression_sin_phi': 10.83420467376709, 'Regression_cos_phi': 11.205184936523438, 'Regression_energy': 10.424720764160156} +rank0 [2026-11-14 05:35:15,762] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 05:35:15,762] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:36:05,181] INFO : Step 1000/500000 rank0 | Train Loss: 14.5445 | Train Loss Unweighted: 5.3518 | Train Loss Components: Classification: 0.0596 | Classification_binary: 4.4874 | Regression_cos_phi: 0.0198 | Regression_energy: 0.3652 | Regression_eta: 0.0245 | Regression_pt: 0.3693 | Regression_sin_phi: 0.0259 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.6678 | Classification_binary: 4.4874 | Regression_cos_phi: 0.2215 | Regression_energy: 3.8075 | Regression_eta: 0.1933 | Regression_pt: 4.8862 | Regression_sin_phi: 0.2807 | LR: 2.01e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.6164s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0029 | max_backward: 0.0404 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.2301 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2564.5 | uss_MB: 1405.3 | child_rss_MB: 22472.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:36:19,136] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 05:36:19,136] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:21,686] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 05:37:21,687] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:57,018] INFO : Step 2000/500000 rank0 | Train Loss: 12.5979 | Train Loss Unweighted: 5.1177 | Train Loss Components: Classification: 0.0403 | Classification_binary: 4.4129 | Regression_cos_phi: 0.0201 | Regression_energy: 0.2992 | Regression_eta: 0.0261 | Regression_pt: 0.3028 | Regression_sin_phi: 0.0163 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4518 | Classification_binary: 4.4129 | Regression_cos_phi: 0.2253 | Regression_energy: 3.1191 | Regression_eta: 0.2055 | Regression_pt: 4.0064 | Regression_sin_phi: 0.1769 | LR: 2.02e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.5733s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0028 | max_backward: 0.0422 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1864 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2566.1 | uss_MB: 1407.0 | child_rss_MB: 22475.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:37:57,020] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:38:24,803] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 05:38:24,803] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:27,775] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 05:39:27,776] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:49,291] INFO : Step 3000/500000 rank0 | Train Loss: 9.4411 | Train Loss Unweighted: 4.5168 | Train Loss Components: Classification: 0.0423 | Classification_binary: 4.0532 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1941 | Regression_eta: 0.0130 | Regression_pt: 0.1937 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4741 | Classification_binary: 4.0532 | Regression_cos_phi: 0.1151 | Regression_energy: 2.0239 | Regression_eta: 0.1029 | Regression_pt: 2.5624 | Regression_sin_phi: 0.1095 | LR: 2.05e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.5798s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0383 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0671 | max_tensorboard_logging: 0.1965 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.0 | uss_MB: 1411.3 | child_rss_MB: 22473.7 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:40:30,507] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 05:40:30,507] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:32,985] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 05:41:32,985] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:40,864] INFO : Step 4000/500000 rank0 | Train Loss: 9.8202 | Train Loss Unweighted: 4.3056 | Train Loss Components: Classification: 0.0490 | Classification_binary: 3.7862 | Regression_cos_phi: 0.0174 | Regression_energy: 0.2147 | Regression_eta: 0.0130 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0131 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.5491 | Classification_binary: 3.7862 | Regression_cos_phi: 0.1945 | Regression_energy: 2.2387 | Regression_eta: 0.1024 | Regression_pt: 2.8070 | Regression_sin_phi: 0.1424 | LR: 2.08e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5806s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0409 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1961 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.0 | uss_MB: 1411.3 | child_rss_MB: 22474.9 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:41:40,866] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:42:36,201] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 05:42:36,201] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:43:33,156] INFO : Step 5000/500000 rank0 | Train Loss: 9.9054 | Train Loss Unweighted: 4.6113 | Train Loss Components: Classification: 0.0357 | Classification_binary: 4.1123 | Regression_cos_phi: 0.0157 | Regression_energy: 0.2079 | Regression_eta: 0.0167 | Regression_pt: 0.2095 | Regression_sin_phi: 0.0135 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3998 | Classification_binary: 4.1123 | Regression_cos_phi: 0.1762 | Regression_energy: 2.1670 | Regression_eta: 0.1320 | Regression_pt: 2.7715 | Regression_sin_phi: 0.1464 | LR: 2.13e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5878s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1979 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2568.3 | uss_MB: 1409.6 | child_rss_MB: 22475.8 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:43:39,613] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 05:43:39,613] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:41,674] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 05:44:41,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:45:24,422] INFO : Step 6000/500000 rank0 | Train Loss: 10.6283 | Train Loss Unweighted: 4.5432 | Train Loss Components: Classification: 0.0393 | Classification_binary: 3.9723 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2514 | Regression_eta: 0.0116 | Regression_pt: 0.2460 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4403 | Classification_binary: 3.9723 | Regression_cos_phi: 0.1186 | Regression_energy: 2.6212 | Regression_eta: 0.0912 | Regression_pt: 3.2550 | Regression_sin_phi: 0.1297 | LR: 2.19e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.6233s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0403 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.2328 | max_tensorboard_flush: 0.0005 | Memory: rss_MB: 2568.3 | uss_MB: 1409.6 | child_rss_MB: 22476.4 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:45:24,424] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:45:44,905] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 05:45:44,905] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:49,962] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 05:46:49,962] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:47:19,174] INFO : Step 7000/500000 rank0 | Train Loss: 10.2920 | Train Loss Unweighted: 4.7103 | Train Loss Components: Classification: 0.0412 | Classification_binary: 4.1889 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2212 | Regression_eta: 0.0089 | Regression_pt: 0.2305 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4609 | Classification_binary: 4.1889 | Regression_cos_phi: 0.1026 | Regression_energy: 2.3064 | Regression_eta: 0.0700 | Regression_pt: 3.0492 | Regression_sin_phi: 0.1139 | LR: 2.26e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.6156s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0029 | max_backward: 0.0403 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.2092 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2568.9 | uss_MB: 1410.2 | child_rss_MB: 22479.0 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:47:53,183] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 05:47:53,183] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:56,150] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 05:48:56,151] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:49:11,298] INFO : Step 8000/500000 rank0 | Train Loss: 9.1124 | Train Loss Unweighted: 4.7340 | Train Loss Components: Classification: 0.0384 | Classification_binary: 4.3234 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1744 | Regression_eta: 0.0064 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4307 | Classification_binary: 4.3234 | Regression_cos_phi: 0.1017 | Regression_energy: 1.8178 | Regression_eta: 0.0503 | Regression_pt: 2.2871 | Regression_sin_phi: 0.1014 | LR: 2.34e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.5642s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0367 | max_loss: 0.0029 | max_backward: 0.0411 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1798 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2568.9 | uss_MB: 1410.2 | child_rss_MB: 22475.3 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:49:11,300] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:49:11,376] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:49:58,172] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 05:49:58,172] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:00,519] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 05:51:00,519] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:01,915] INFO : Step 9000/500000 rank0 | Train Loss: 9.5025 | Train Loss Unweighted: 4.3462 | Train Loss Components: Classification: 0.0408 | Classification_binary: 3.8611 | Regression_cos_phi: 0.0132 | Regression_energy: 0.2007 | Regression_eta: 0.0129 | Regression_pt: 0.2032 | Regression_sin_phi: 0.0142 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4568 | Classification_binary: 3.8611 | Regression_cos_phi: 0.1482 | Regression_energy: 2.0924 | Regression_eta: 0.1017 | Regression_pt: 2.6885 | Regression_sin_phi: 0.1538 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.5864s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1988 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2568.0 | uss_MB: 1409.3 | child_rss_MB: 22476.5 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:52:03,512] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 05:52:03,512] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:53,569] INFO : Step 10000/500000 rank0 | Train Loss: 9.6285 | Train Loss Unweighted: 4.3744 | Train Loss Components: Classification: 0.0381 | Classification_binary: 3.8801 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2064 | Regression_eta: 0.0161 | Regression_pt: 0.2120 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4268 | Classification_binary: 3.8801 | Regression_cos_phi: 0.1191 | Regression_energy: 2.1517 | Regression_eta: 0.1272 | Regression_pt: 2.8053 | Regression_sin_phi: 0.1182 | LR: 2.52e-05 | DataLoad Time: 0.0019s | Model Forward Time: 0.5763s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0028 | max_backward: 0.0399 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1976 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2568.5 | uss_MB: 1409.8 | child_rss_MB: 22477.5 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:52:53,571] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:52:53,646] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:52:53,648] INFO : Running validation on rank0 +rank0 [2026-11-14 05:52:53,648] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:52:53,648] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:52:53,909] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 05:52:54,138] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:52:54,221] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:52:54,275] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:52:54,331] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:52:54,377] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:52:54,432] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:52:54,486] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:52:54,539] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:52:54,617] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:52:54,695] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:52:54,750] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:52:54,806] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:52:54,854] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:52:54,910] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:52:54,966] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:52:55,017] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:52:55,021] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:52:55,023] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:52:55,025] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:52:55,028] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:52:55,030] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:52:55,033] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:52:55,035] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:52:55,038] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:52:55,040] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:52:55,042] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:52:55,045] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:52:55,047] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:52:55,049] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:52:55,052] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:52:55,054] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:52:56,631] INFO : VALIDATION | Step=10000/500000 | Train Loss=9.6285 | Valid Loss=9.2713 | Train Loss Unweighted=4.3744 | Valid Loss Unweighted=4.3800 | Stale=0 | ETA=919.9m +rank0 [2026-11-14 05:52:56,631] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:52:57,134] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:52:57,145] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:57,201] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:52:58,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:52:58,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:52:58,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:52:58,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:52:59,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:52:59,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:52:59,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:52:59,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:53:00,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:53:00,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:53:00,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:53:00,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:53:01,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:53:01,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:53:01,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:53:01,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:53:02,076] INFO : Time taken to make predictions on device 0 is: 0.08 min +rank0 [2026-11-14 05:53:02,085] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:53:02,673] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:53:03,794] INFO : Plotted number of elements +rank0 [2026-11-14 05:53:05,198] INFO : Plotted jets +rank0 [2026-11-14 05:53:06,204] INFO : Plotted jet ratio +rank0 [2026-11-14 05:53:19,623] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 05:53:19,623] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:22,550] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 05:54:22,551] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:59,042] INFO : Step 11000/500000 rank0 | Train Loss: 10.1912 | Train Loss Unweighted: 4.5144 | Train Loss Components: Classification: 0.0474 | Classification_binary: 3.9782 | Regression_cos_phi: 0.0167 | Regression_energy: 0.2158 | Regression_eta: 0.0167 | Regression_pt: 0.2161 | Regression_sin_phi: 0.0234 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.5314 | Classification_binary: 3.9782 | Regression_cos_phi: 0.1873 | Regression_energy: 2.2499 | Regression_eta: 0.1316 | Regression_pt: 2.8596 | Regression_sin_phi: 0.2532 | LR: 2.63e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5826s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0027 | max_backward: 0.0394 | max_optimizer_step: 0.0709 | max_tensorboard_logging: 0.1775 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.5 | uss_MB: 1599.4 | child_rss_MB: 22482.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 24.3 +rank0 [2026-11-14 05:55:25,821] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 05:55:25,821] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:28,094] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 05:56:28,095] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:50,624] INFO : Step 12000/500000 rank0 | Train Loss: 9.2129 | Train Loss Unweighted: 4.1807 | Train Loss Components: Classification: 0.0340 | Classification_binary: 3.7061 | Regression_cos_phi: 0.0159 | Regression_energy: 0.2010 | Regression_eta: 0.0133 | Regression_pt: 0.1954 | Regression_sin_phi: 0.0149 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3812 | Classification_binary: 3.7061 | Regression_cos_phi: 0.1779 | Regression_energy: 2.0955 | Regression_eta: 0.1048 | Regression_pt: 2.5858 | Regression_sin_phi: 0.1616 | LR: 2.75e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.5800s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0383 | max_loss: 0.0030 | max_backward: 0.0405 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.1849 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2744.1 | uss_MB: 1600.1 | child_rss_MB: 22484.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:56:50,626] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:56:50,704] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:57:31,172] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 05:57:31,172] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:33,609] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 05:58:33,609] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:42,192] INFO : Step 13000/500000 rank0 | Train Loss: 10.9813 | Train Loss Unweighted: 4.4905 | Train Loss Components: Classification: 0.0406 | Classification_binary: 3.8827 | Regression_cos_phi: 0.0105 | Regression_energy: 0.2655 | Regression_eta: 0.0134 | Regression_pt: 0.2685 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4544 | Classification_binary: 3.8827 | Regression_cos_phi: 0.1181 | Regression_energy: 2.7676 | Regression_eta: 0.1053 | Regression_pt: 3.5526 | Regression_sin_phi: 0.1005 | LR: 2.88e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.5990s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0357 | max_loss: 0.0036 | max_backward: 0.0409 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.2010 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2669.8 | uss_MB: 1525.8 | child_rss_MB: 22482.4 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:59:36,571] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:59:36,571] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:34,422] INFO : Step 14000/500000 rank0 | Train Loss: 8.9930 | Train Loss Unweighted: 4.3760 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.9433 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1805 | Regression_eta: 0.0115 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3349 | Classification_binary: 3.9433 | Regression_cos_phi: 0.1181 | Regression_energy: 1.8820 | Regression_eta: 0.0903 | Regression_pt: 2.5110 | Regression_sin_phi: 0.1133 | LR: 3.02e-05 | DataLoad Time: 0.0020s | Model Forward Time: 0.6538s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0401 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.2671 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2669.8 | uss_MB: 1525.8 | child_rss_MB: 22484.2 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:00:34,424] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:00:34,504] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 06:00:40,115] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 06:00:40,115] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:42,725] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 06:01:42,725] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:26,826] INFO : Step 15000/500000 rank0 | Train Loss: 9.9478 | Train Loss Unweighted: 4.2329 | Train Loss Components: Classification: 0.0436 | Classification_binary: 3.6984 | Regression_cos_phi: 0.0128 | Regression_energy: 0.2210 | Regression_eta: 0.0104 | Regression_pt: 0.2334 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4879 | Classification_binary: 3.6984 | Regression_cos_phi: 0.1432 | Regression_energy: 2.3034 | Regression_eta: 0.0823 | Regression_pt: 3.0881 | Regression_sin_phi: 0.1446 | LR: 3.17e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.5951s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.2059 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2672.6 | uss_MB: 1528.6 | child_rss_MB: 22483.6 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:02:46,245] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 06:02:46,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:48,811] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 06:03:48,811] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:18,668] INFO : Step 16000/500000 rank0 | Train Loss: 8.6107 | Train Loss Unweighted: 4.0345 | Train Loss Components: Classification: 0.0365 | Classification_binary: 3.6061 | Regression_cos_phi: 0.0140 | Regression_energy: 0.1788 | Regression_eta: 0.0064 | Regression_pt: 0.1822 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4091 | Classification_binary: 3.6061 | Regression_cos_phi: 0.1569 | Regression_energy: 1.8643 | Regression_eta: 0.0506 | Regression_pt: 2.4107 | Regression_sin_phi: 0.1129 | LR: 3.33e-05 | DataLoad Time: 0.0022s | Model Forward Time: 0.5419s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0413 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.1560 | max_tensorboard_flush: 0.0006 | Memory: rss_MB: 2672.6 | uss_MB: 1528.6 | child_rss_MB: 22483.6 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:04:18,670] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:04:18,747] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 06:04:52,138] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 06:04:52,139] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:54,835] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 06:05:54,836] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:06:10,902] INFO : Step 17000/500000 rank0 | Train Loss: 9.3705 | Train Loss Unweighted: 4.0817 | Train Loss Components: Classification: 0.0335 | Classification_binary: 3.5859 | Regression_cos_phi: 0.0125 | Regression_energy: 0.2134 | Regression_eta: 0.0091 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0143 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3753 | Classification_binary: 3.5859 | Regression_cos_phi: 0.1404 | Regression_energy: 2.2250 | Regression_eta: 0.0714 | Regression_pt: 2.8175 | Regression_sin_phi: 0.1551 | LR: 3.51e-05 | DataLoad Time: 0.0020s | Model Forward Time: 0.5792s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1853 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2668.9 | uss_MB: 1524.9 | child_rss_MB: 22483.8 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:58,296] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 06:06:58,296] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:08:01,575] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 06:08:01,575] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:08:03,854] INFO : Step 18000/500000 rank0 | Train Loss: 9.2232 | Train Loss Unweighted: 4.2970 | Train Loss Components: Classification: 0.0388 | Classification_binary: 3.8345 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1941 | Regression_eta: 0.0108 | Regression_pt: 0.1970 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4349 | Classification_binary: 3.8345 | Regression_cos_phi: 0.1055 | Regression_energy: 2.0237 | Regression_eta: 0.0852 | Regression_pt: 2.6063 | Regression_sin_phi: 0.1331 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.5764s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0362 | max_loss: 0.0030 | max_backward: 0.0407 | max_optimizer_step: 0.0691 | max_tensorboard_logging: 0.1826 | max_tensorboard_flush: 0.0005 | Memory: rss_MB: 2670.1 | uss_MB: 1526.1 | child_rss_MB: 22484.4 | allocated_MB: 126.1 | reserved_MB: 650.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 06:08:03,856] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:08:03,934] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 06:09:07,424] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 06:09:07,424] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:58,912] INFO : Step 19000/500000 rank0 | Train Loss: 9.7826 | Train Loss Unweighted: 4.5104 | Train Loss Components: Classification: 0.0381 | Classification_binary: 4.0161 | Regression_cos_phi: 0.0101 | Regression_energy: 0.2145 | Regression_eta: 0.0093 | Regression_pt: 0.2121 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4266 | Classification_binary: 4.0161 | Regression_cos_phi: 0.1134 | Regression_energy: 2.2361 | Regression_eta: 0.0731 | Regression_pt: 2.8059 | Regression_sin_phi: 0.1114 | LR: 3.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5961s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0031 | max_backward: 0.0402 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.2071 | max_tensorboard_flush: 0.0005 | Memory: rss_MB: 2672.7 | uss_MB: 1528.7 | child_rss_MB: 22484.0 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:10:10,974] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 06:10:10,974] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:13,764] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 06:11:13,764] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:53,053] INFO : Step 20000/500000 rank0 | Train Loss: 8.9256 | Train Loss Unweighted: 4.1463 | Train Loss Components: Classification: 0.0414 | Classification_binary: 3.6990 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1840 | Regression_eta: 0.0083 | Regression_pt: 0.1926 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4640 | Classification_binary: 3.6990 | Regression_cos_phi: 0.1162 | Regression_energy: 1.9181 | Regression_eta: 0.0657 | Regression_pt: 2.5486 | Regression_sin_phi: 0.1140 | LR: 4.07e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.6120s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0030 | max_backward: 0.0403 | max_optimizer_step: 0.0679 | max_tensorboard_logging: 0.2328 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2672.9 | uss_MB: 1528.8 | child_rss_MB: 22484.6 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 24.3 +rank0 [2026-11-14 06:11:53,079] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:11:53,191] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:11:53,192] INFO : Running validation on rank0 +rank0 [2026-11-14 06:11:53,193] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:11:53,193] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:11:53,346] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 06:11:53,464] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:11:53,595] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:11:53,665] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:11:53,733] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:11:53,798] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:11:53,864] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:11:53,932] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:11:54,030] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:11:54,145] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:11:54,257] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:11:54,325] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:11:54,391] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:11:54,456] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:11:54,522] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:11:54,587] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:11:54,683] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:11:54,688] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:11:54,690] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:11:54,693] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:11:54,695] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:11:54,697] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:11:54,699] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:11:54,702] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:11:54,704] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:11:54,706] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:11:54,708] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:11:54,710] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:11:54,713] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:11:54,715] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:11:54,717] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:11:54,719] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:11:56,314] INFO : VALIDATION | Step=20000/500000 | Train Loss=8.9256 | Valid Loss=8.9078 | Train Loss Unweighted=4.1463 | Valid Loss Unweighted=4.2175 | Stale=0 | ETA=906.4m +rank0 [2026-11-14 06:11:56,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:11:56,974] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:11:57,003] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:57,057] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:11:57,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:11:57,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:11:58,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:11:58,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:11:58,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:11:58,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:11:59,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:11:59,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:11:59,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:11:59,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:12:00,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:12:00,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:12:00,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:12:00,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:12:01,107] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:12:01,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:12:01,392] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:12:01,402] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:12:01,991] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:12:02,389] INFO : Plotted number of elements +rank0 [2026-11-14 06:12:03,719] INFO : Plotted jets +rank0 [2026-11-14 06:12:04,701] INFO : Plotted jet ratio +rank0 [2026-11-14 06:12:30,938] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 06:12:30,938] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:33,271] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 06:13:33,271] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:56,444] INFO : Step 21000/500000 rank0 | Train Loss: 7.9236 | Train Loss Unweighted: 3.8248 | Train Loss Components: Classification: 0.0331 | Classification_binary: 3.4391 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1610 | Regression_eta: 0.0084 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0135 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3705 | Classification_binary: 3.4391 | Regression_cos_phi: 0.1241 | Regression_energy: 1.6788 | Regression_eta: 0.0664 | Regression_pt: 2.0986 | Regression_sin_phi: 0.1462 | LR: 4.28e-05 | DataLoad Time: 0.0027s | Model Forward Time: 0.5509s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0028 | max_backward: 0.0438 | max_optimizer_step: 0.0689 | max_tensorboard_logging: 0.1501 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2699.6 | uss_MB: 1555.6 | child_rss_MB: 22486.9 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:14:35,667] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 06:14:35,667] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:37,834] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 06:15:37,834] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:47,321] INFO : Step 22000/500000 rank0 | Train Loss: 10.8922 | Train Loss Unweighted: 4.3936 | Train Loss Components: Classification: 0.0329 | Classification_binary: 3.7828 | Regression_cos_phi: 0.0098 | Regression_energy: 0.2782 | Regression_eta: 0.0147 | Regression_pt: 0.2645 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3682 | Classification_binary: 3.7828 | Regression_cos_phi: 0.1100 | Regression_energy: 2.8998 | Regression_eta: 0.1162 | Regression_pt: 3.4995 | Regression_sin_phi: 0.1157 | LR: 4.50e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5989s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0404 | max_optimizer_step: 0.0692 | max_tensorboard_logging: 0.2073 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2700.3 | uss_MB: 1556.4 | child_rss_MB: 22487.4 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:15:47,323] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:15:47,403] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:16:40,836] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 06:16:40,836] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:39,589] INFO : Step 23000/500000 rank0 | Train Loss: 8.9377 | Train Loss Unweighted: 4.3616 | Train Loss Components: Classification: 0.0308 | Classification_binary: 3.9318 | Regression_cos_phi: 0.0132 | Regression_energy: 0.1818 | Regression_eta: 0.0103 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3450 | Classification_binary: 3.9318 | Regression_cos_phi: 0.1477 | Regression_energy: 1.8950 | Regression_eta: 0.0814 | Regression_pt: 2.4197 | Regression_sin_phi: 0.1172 | LR: 4.73e-05 | DataLoad Time: 0.0023s | Model Forward Time: 0.5848s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0410 | max_loss: 0.0029 | max_backward: 0.0411 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1898 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2703.4 | uss_MB: 1559.4 | child_rss_MB: 22486.1 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:17:44,250] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 06:17:44,250] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:46,997] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 06:18:46,997] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:19:31,680] INFO : Step 24000/500000 rank0 | Train Loss: 7.4872 | Train Loss Unweighted: 3.8002 | Train Loss Components: Classification: 0.0332 | Classification_binary: 3.4546 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1482 | Regression_eta: 0.0055 | Regression_pt: 0.1467 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3723 | Classification_binary: 3.4546 | Regression_cos_phi: 0.0551 | Regression_energy: 1.5454 | Regression_eta: 0.0433 | Regression_pt: 1.9404 | Regression_sin_phi: 0.0761 | LR: 4.97e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.5598s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0026 | max_backward: 0.0396 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1727 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2703.5 | uss_MB: 1559.5 | child_rss_MB: 22487.6 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:19:31,682] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:19:31,764] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:19:50,317] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 06:19:50,317] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:52,988] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 06:20:52,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:21:24,019] INFO : Step 25000/500000 rank0 | Train Loss: 9.7542 | Train Loss Unweighted: 4.1255 | Train Loss Components: Classification: 0.0351 | Classification_binary: 3.5990 | Regression_cos_phi: 0.0170 | Regression_energy: 0.2231 | Regression_eta: 0.0097 | Regression_pt: 0.2301 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3933 | Classification_binary: 3.5990 | Regression_cos_phi: 0.1900 | Regression_energy: 2.3260 | Regression_eta: 0.0765 | Regression_pt: 3.0448 | Regression_sin_phi: 0.1246 | LR: 5.22e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.6089s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0363 | max_loss: 0.0030 | max_backward: 0.0405 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.2093 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2699.6 | uss_MB: 1555.7 | child_rss_MB: 22486.9 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:21:56,685] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 06:21:56,685] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:59,499] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 06:22:59,499] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:16,387] INFO : Step 26000/500000 rank0 | Train Loss: 8.5531 | Train Loss Unweighted: 4.1961 | Train Loss Components: Classification: 0.0357 | Classification_binary: 3.7891 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1742 | Regression_eta: 0.0062 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3999 | Classification_binary: 3.7891 | Regression_cos_phi: 0.0585 | Regression_energy: 1.8160 | Regression_eta: 0.0486 | Regression_pt: 2.3683 | Regression_sin_phi: 0.0727 | LR: 5.47e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.6198s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0029 | max_backward: 0.0422 | max_optimizer_step: 0.0689 | max_tensorboard_logging: 0.2333 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2700.3 | uss_MB: 1556.3 | child_rss_MB: 22487.1 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:23:16,389] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:23:16,472] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:24:02,508] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 06:24:02,508] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:05,450] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 06:25:05,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:08,640] INFO : Step 27000/500000 rank0 | Train Loss: 7.7604 | Train Loss Unweighted: 3.8876 | Train Loss Components: Classification: 0.0353 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1500 | Regression_eta: 0.0079 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3957 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0638 | Regression_energy: 1.5633 | Regression_eta: 0.0623 | Regression_pt: 2.0607 | Regression_sin_phi: 0.0899 | LR: 5.74e-05 | DataLoad Time: 0.0027s | Model Forward Time: 0.5751s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0365 | max_loss: 0.0029 | max_backward: 0.0407 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.1758 | max_tensorboard_flush: 0.0005 | Memory: rss_MB: 2703.4 | uss_MB: 1559.4 | child_rss_MB: 22485.5 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:26:09,410] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 06:26:09,410] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:27:01,561] INFO : Step 28000/500000 rank0 | Train Loss: 8.1155 | Train Loss Unweighted: 4.0403 | Train Loss Components: Classification: 0.0352 | Classification_binary: 3.6588 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1565 | Regression_eta: 0.0072 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3946 | Classification_binary: 3.6588 | Regression_cos_phi: 0.1163 | Regression_energy: 1.6315 | Regression_eta: 0.0565 | Regression_pt: 2.1656 | Regression_sin_phi: 0.0922 | LR: 6.01e-05 | DataLoad Time: 0.0020s | Model Forward Time: 0.5889s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0388 | max_loss: 0.0029 | max_backward: 0.0409 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1903 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2703.6 | uss_MB: 1559.7 | child_rss_MB: 22487.4 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:27:01,563] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:27:01,636] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:27:12,825] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 06:27:12,825] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:15,680] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 06:28:15,681] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:53,796] INFO : Step 29000/500000 rank0 | Train Loss: 8.1836 | Train Loss Unweighted: 3.8308 | Train Loss Components: Classification: 0.0368 | Classification_binary: 3.4204 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1687 | Regression_eta: 0.0143 | Regression_pt: 0.1715 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4120 | Classification_binary: 3.4204 | Regression_cos_phi: 0.1241 | Regression_energy: 1.7590 | Regression_eta: 0.1125 | Regression_pt: 2.2695 | Regression_sin_phi: 0.0862 | LR: 6.29e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5854s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0030 | max_backward: 0.0453 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1819 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2703.6 | uss_MB: 1559.7 | child_rss_MB: 22486.2 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:29:18,900] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 06:29:18,900] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:21,790] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 06:30:21,790] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:47,311] INFO : Step 30000/500000 rank0 | Train Loss: 8.5306 | Train Loss Unweighted: 4.0331 | Train Loss Components: Classification: 0.0327 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1803 | Regression_eta: 0.0097 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3660 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0920 | Regression_energy: 1.8792 | Regression_eta: 0.0762 | Regression_pt: 2.4215 | Regression_sin_phi: 0.0843 | LR: 6.58e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.6339s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0032 | max_backward: 0.0416 | max_optimizer_step: 0.0718 | max_tensorboard_logging: 0.2206 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2700.3 | uss_MB: 1556.3 | child_rss_MB: 22482.7 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:30:47,313] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:30:47,396] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:30:47,397] INFO : Running validation on rank0 +rank0 [2026-11-14 06:30:47,397] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:30:47,397] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:30:47,592] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 06:30:47,720] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:30:47,843] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:30:47,917] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:30:47,988] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:30:48,058] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:30:48,130] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:30:48,201] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:30:48,304] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:30:48,426] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:30:48,545] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:30:48,615] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:30:48,686] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:30:48,754] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:30:48,824] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:30:48,895] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:30:49,001] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:30:49,006] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:30:49,009] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:30:49,012] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:30:49,014] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:30:49,016] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:30:49,019] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:30:49,021] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:30:49,024] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:30:49,026] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:30:49,028] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:30:49,030] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:30:49,033] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:30:49,035] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:30:49,037] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:30:49,040] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:30:50,702] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.5306 | Valid Loss=8.6071 | Train Loss Unweighted=4.0331 | Valid Loss Unweighted=4.0722 | Stale=0 | ETA=887.9m +rank0 [2026-11-14 06:30:50,703] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:30:51,215] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:30:51,228] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:51,283] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:30:51,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:30:52,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:30:52,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:30:52,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:30:52,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:30:53,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:30:53,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:30:53,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:30:53,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:30:54,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:30:54,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:30:54,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:30:55,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:30:55,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:30:55,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:30:55,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:30:55,902] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:30:55,924] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:56,548] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:56,977] INFO : Plotted number of elements +rank0 [2026-11-14 06:30:58,417] INFO : Plotted jets +rank0 [2026-11-14 06:30:59,492] INFO : Plotted jet ratio +rank0 [2026-11-14 06:31:40,791] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 06:31:40,791] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:44,006] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 06:32:44,006] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:54,695] INFO : Step 31000/500000 rank0 | Train Loss: 8.3726 | Train Loss Unweighted: 3.9560 | Train Loss Components: Classification: 0.0371 | Classification_binary: 3.5384 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1659 | Regression_eta: 0.0193 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4157 | Classification_binary: 3.5384 | Regression_cos_phi: 0.1149 | Regression_energy: 1.7291 | Regression_eta: 0.1521 | Regression_pt: 2.3057 | Regression_sin_phi: 0.1167 | LR: 6.88e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.6486s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.2221 | max_tensorboard_flush: 0.0006 | Memory: rss_MB: 2825.6 | uss_MB: 1681.7 | child_rss_MB: 22484.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:33:47,597] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 06:33:47,597] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:47,096] INFO : Step 32000/500000 rank0 | Train Loss: 9.0571 | Train Loss Unweighted: 4.1940 | Train Loss Components: Classification: 0.0326 | Classification_binary: 3.7385 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1937 | Regression_eta: 0.0081 | Regression_pt: 0.1970 | Regression_sin_phi: 0.0148 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3648 | Classification_binary: 3.7385 | Regression_cos_phi: 0.1037 | Regression_energy: 2.0198 | Regression_eta: 0.0637 | Regression_pt: 2.6063 | Regression_sin_phi: 0.1603 | LR: 7.19e-05 | DataLoad Time: 0.0024s | Model Forward Time: 0.5816s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0032 | max_backward: 0.0414 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1771 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2825.6 | uss_MB: 1681.7 | child_rss_MB: 22485.4 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:34:47,098] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:34:47,174] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:34:50,988] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 06:34:50,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:54,456] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 06:35:54,456] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:39,870] INFO : Step 33000/500000 rank0 | Train Loss: 6.9426 | Train Loss Unweighted: 3.4761 | Train Loss Components: Classification: 0.0268 | Classification_binary: 3.1515 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1395 | Regression_eta: 0.0067 | Regression_pt: 0.1414 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3001 | Classification_binary: 3.1515 | Regression_cos_phi: 0.0587 | Regression_energy: 1.4546 | Regression_eta: 0.0527 | Regression_pt: 1.8712 | Regression_sin_phi: 0.0537 | LR: 7.51e-05 | DataLoad Time: 0.0013s | Model Forward Time: 0.5782s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0027 | max_backward: 0.0398 | max_optimizer_step: 0.0691 | max_tensorboard_logging: 0.1851 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2724.8 | uss_MB: 1580.9 | child_rss_MB: 22487.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:36:57,296] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 06:36:57,296] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:59,642] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 06:37:59,642] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:31,032] INFO : Step 34000/500000 rank0 | Train Loss: 7.4224 | Train Loss Unweighted: 3.5897 | Train Loss Components: Classification: 0.0325 | Classification_binary: 3.2303 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1525 | Regression_eta: 0.0082 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3644 | Classification_binary: 3.2303 | Regression_cos_phi: 0.0850 | Regression_energy: 1.5900 | Regression_eta: 0.0646 | Regression_pt: 2.0441 | Regression_sin_phi: 0.0441 | LR: 7.83e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.5427s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0026 | max_backward: 0.0395 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1538 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2725.5 | uss_MB: 1581.6 | child_rss_MB: 22486.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:38:31,034] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:38:31,107] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:39:02,489] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 06:39:02,489] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:05,167] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 06:40:05,167] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:23,389] INFO : Step 35000/500000 rank0 | Train Loss: 7.5492 | Train Loss Unweighted: 3.7288 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.3725 | Regression_cos_phi: 0.0046 | Regression_energy: 0.1562 | Regression_eta: 0.0045 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3037 | Classification_binary: 3.3725 | Regression_cos_phi: 0.0520 | Regression_energy: 1.6286 | Regression_eta: 0.0357 | Regression_pt: 2.1099 | Regression_sin_phi: 0.0469 | LR: 8.16e-05 | DataLoad Time: 0.0018s | Model Forward Time: 0.5924s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.2036 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2729.6 | uss_MB: 1585.7 | child_rss_MB: 22485.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:41:08,318] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 06:41:08,319] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:11,703] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 06:42:11,703] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:15,712] INFO : Step 36000/500000 rank0 | Train Loss: 8.5782 | Train Loss Unweighted: 3.7415 | Train Loss Components: Classification: 0.0333 | Classification_binary: 3.2914 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1966 | Regression_eta: 0.0035 | Regression_pt: 0.2034 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3729 | Classification_binary: 3.2914 | Regression_cos_phi: 0.0742 | Regression_energy: 2.0496 | Regression_eta: 0.0278 | Regression_pt: 2.6914 | Regression_sin_phi: 0.0708 | LR: 8.50e-05 | DataLoad Time: 0.0023s | Model Forward Time: 0.5265s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1334 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2729.7 | uss_MB: 1585.8 | child_rss_MB: 22488.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:42:15,714] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:42:15,790] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:43:15,194] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 06:43:15,194] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:44:08,035] INFO : Step 37000/500000 rank0 | Train Loss: 6.8225 | Train Loss Unweighted: 3.6000 | Train Loss Components: Classification: 0.0235 | Classification_binary: 3.2978 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1269 | Regression_eta: 0.0059 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2634 | Classification_binary: 3.2978 | Regression_cos_phi: 0.0807 | Regression_energy: 1.3232 | Regression_eta: 0.0469 | Regression_pt: 1.7059 | Regression_sin_phi: 0.1046 | LR: 8.85e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5885s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0032 | max_backward: 0.0407 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.1894 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2729.7 | uss_MB: 1585.8 | child_rss_MB: 22488.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:44:18,369] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 06:44:18,369] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:45:21,713] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 06:45:21,714] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:46:01,065] INFO : Step 38000/500000 rank0 | Train Loss: 7.9930 | Train Loss Unweighted: 3.6223 | Train Loss Components: Classification: 0.0309 | Classification_binary: 3.2121 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1773 | Regression_eta: 0.0078 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0113 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3461 | Classification_binary: 3.2121 | Regression_cos_phi: 0.0951 | Regression_energy: 1.8483 | Regression_eta: 0.0613 | Regression_pt: 2.3074 | Regression_sin_phi: 0.1227 | LR: 9.21e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5801s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0413 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1717 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2725.6 | uss_MB: 1581.7 | child_rss_MB: 22487.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:46:01,067] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:46:01,140] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:46:25,047] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 06:46:25,047] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:28,311] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 06:47:28,312] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:53,681] INFO : Step 39000/500000 rank0 | Train Loss: 9.5954 | Train Loss Unweighted: 4.3674 | Train Loss Components: Classification: 0.0336 | Classification_binary: 3.8771 | Regression_cos_phi: 0.0085 | Regression_energy: 0.2139 | Regression_eta: 0.0114 | Regression_pt: 0.2136 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3763 | Classification_binary: 3.8771 | Regression_cos_phi: 0.0952 | Regression_energy: 2.2303 | Regression_eta: 0.0896 | Regression_pt: 2.8263 | Regression_sin_phi: 0.1005 | LR: 9.57e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.6071s | Phase Times: max_pre_step_barrier: 0.0005 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0679 | max_tensorboard_logging: 0.2215 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 2729.6 | uss_MB: 1585.8 | child_rss_MB: 22489.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:48:31,459] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 06:48:31,459] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:34,413] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 06:49:34,413] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:45,961] INFO : Step 40000/500000 rank0 | Train Loss: 9.1555 | Train Loss Unweighted: 4.1032 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.6323 | Regression_cos_phi: 0.0045 | Regression_energy: 0.2108 | Regression_eta: 0.0060 | Regression_pt: 0.2142 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3113 | Classification_binary: 3.6323 | Regression_cos_phi: 0.0504 | Regression_energy: 2.1972 | Regression_eta: 0.0473 | Regression_pt: 2.8336 | Regression_sin_phi: 0.0834 | LR: 9.94e-05 | DataLoad Time: 0.0019s | Model Forward Time: 0.6079s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0398 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.2129 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.0 | uss_MB: 1586.1 | child_rss_MB: 22490.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:49:45,964] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 06:49:46,050] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:49:46,053] INFO : Running validation on rank0 +rank0 [2026-11-14 06:49:46,053] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:49:46,053] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:49:46,255] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 06:49:46,391] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:49:46,513] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:49:46,587] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:49:46,657] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:49:46,725] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:49:46,794] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:49:46,865] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:49:46,965] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:49:47,085] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:49:47,200] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:49:47,269] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:49:47,338] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:49:47,406] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:49:47,474] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:49:47,544] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:49:47,647] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:49:47,652] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:49:47,655] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:49:47,657] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:49:47,659] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:49:47,661] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:49:47,664] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:49:47,666] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:49:47,668] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:49:47,671] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:49:47,673] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:49:47,675] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:49:47,677] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:49:47,680] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:49:47,682] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:49:47,684] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:49:49,301] INFO : VALIDATION | Step=40000/500000 | Train Loss=9.1555 | Valid Loss=8.3071 | Train Loss Unweighted=4.1032 | Valid Loss Unweighted=3.8876 | Stale=0 | ETA=870.0m +rank0 [2026-11-14 06:49:49,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:49:49,769] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:49:49,784] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:49,838] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:49:50,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:49:50,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:49:50,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:49:51,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:49:51,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:49:51,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:49:51,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:49:52,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:49:52,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:49:52,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:49:52,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:49:53,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:49:53,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:49:53,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:49:54,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:49:54,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:49:54,300] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:49:54,309] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:54,897] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:55,309] INFO : Plotted number of elements +rank0 [2026-11-14 06:49:56,677] INFO : Plotted jets +rank0 [2026-11-14 06:49:57,681] INFO : Plotted jet ratio +rank0 [2026-11-14 06:50:49,978] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 06:50:49,978] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:51:50,347] INFO : Step 41000/500000 rank0 | Train Loss: 7.1656 | Train Loss Unweighted: 3.6709 | Train Loss Components: Classification: 0.0302 | Classification_binary: 3.3430 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1376 | Regression_eta: 0.0060 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3384 | Classification_binary: 3.3430 | Regression_cos_phi: 0.0820 | Regression_energy: 1.4344 | Regression_eta: 0.0475 | Regression_pt: 1.8256 | Regression_sin_phi: 0.0947 | LR: 1.03e-04 | DataLoad Time: 0.0021s | Model Forward Time: 0.5875s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0027 | max_backward: 0.0400 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.1883 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2751.3 | uss_MB: 1607.5 | child_rss_MB: 22495.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:51:53,261] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 06:51:53,261] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:52:55,802] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 06:52:55,802] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:53:44,833] INFO : Step 42000/500000 rank0 | Train Loss: 6.9200 | Train Loss Unweighted: 3.4885 | Train Loss Components: Classification: 0.0284 | Classification_binary: 3.1661 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1318 | Regression_eta: 0.0072 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3178 | Classification_binary: 3.1661 | Regression_cos_phi: 0.1230 | Regression_energy: 1.3743 | Regression_eta: 0.0568 | Regression_pt: 1.7810 | Regression_sin_phi: 0.1009 | LR: 1.07e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.6297s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0379 | max_loss: 0.0032 | max_backward: 0.0415 | max_optimizer_step: 0.0725 | max_tensorboard_logging: 0.2135 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2752.0 | uss_MB: 1608.1 | child_rss_MB: 22493.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:53:44,835] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:53:44,919] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:54:02,120] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 06:54:02,120] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:55:05,618] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 06:55:05,618] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:55:38,203] INFO : Step 43000/500000 rank0 | Train Loss: 8.6636 | Train Loss Unweighted: 3.9669 | Train Loss Components: Classification: 0.0307 | Classification_binary: 3.5272 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1916 | Regression_eta: 0.0069 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3438 | Classification_binary: 3.5272 | Regression_cos_phi: 0.1294 | Regression_energy: 1.9976 | Regression_eta: 0.0545 | Regression_pt: 2.5128 | Regression_sin_phi: 0.0982 | LR: 1.11e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.5455s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0372 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1509 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.1 | uss_MB: 1610.2 | child_rss_MB: 22492.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 28.3 +rank0 [2026-11-14 06:56:08,741] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 06:56:08,741] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:57:11,418] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 06:57:11,418] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:57:30,361] INFO : Step 44000/500000 rank0 | Train Loss: 6.6579 | Train Loss Unweighted: 3.4944 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.1944 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1198 | Regression_eta: 0.0159 | Regression_pt: 0.1241 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3096 | Classification_binary: 3.1944 | Regression_cos_phi: 0.0675 | Regression_energy: 1.2490 | Regression_eta: 0.1257 | Regression_pt: 1.6423 | Regression_sin_phi: 0.0695 | LR: 1.15e-04 | DataLoad Time: 0.0017s | Model Forward Time: 0.5799s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0028 | max_backward: 0.0413 | max_optimizer_step: 0.0705 | max_tensorboard_logging: 0.1701 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.8 | uss_MB: 1611.0 | child_rss_MB: 22494.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:57:30,363] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:57:30,441] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:58:14,890] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 06:58:14,890] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:59:17,641] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 06:59:17,641] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:59:22,975] INFO : Step 45000/500000 rank0 | Train Loss: 7.7431 | Train Loss Unweighted: 3.7460 | Train Loss Components: Classification: 0.0264 | Classification_binary: 3.3718 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1599 | Regression_eta: 0.0093 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2961 | Classification_binary: 3.3718 | Regression_cos_phi: 0.0653 | Regression_energy: 1.6673 | Regression_eta: 0.0737 | Regression_pt: 2.1951 | Regression_sin_phi: 0.0738 | LR: 1.19e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5812s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0401 | max_optimizer_step: 0.0696 | max_tensorboard_logging: 0.1811 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2754.9 | uss_MB: 1611.0 | child_rss_MB: 22493.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:00:21,185] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 07:00:21,185] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:01:14,788] INFO : Step 46000/500000 rank0 | Train Loss: 8.0185 | Train Loss Unweighted: 3.7094 | Train Loss Components: Classification: 0.0303 | Classification_binary: 3.3029 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1687 | Regression_eta: 0.0148 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0127 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3391 | Classification_binary: 3.3029 | Regression_cos_phi: 0.0967 | Regression_energy: 1.7591 | Regression_eta: 0.1170 | Regression_pt: 2.2664 | Regression_sin_phi: 0.1374 | LR: 1.23e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.5745s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0030 | max_backward: 0.0413 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1847 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2752.1 | uss_MB: 1608.2 | child_rss_MB: 22495.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 56.3 +rank0 [2026-11-14 07:01:14,790] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:01:14,867] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 07:01:24,164] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 07:01:24,164] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:02:27,047] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 07:02:27,047] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:03:07,337] INFO : Step 47000/500000 rank0 | Train Loss: 6.3793 | Train Loss Unweighted: 3.3890 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.1070 | Regression_cos_phi: 0.0128 | Regression_energy: 0.1133 | Regression_eta: 0.0070 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3360 | Classification_binary: 3.1070 | Regression_cos_phi: 0.1432 | Regression_energy: 1.1811 | Regression_eta: 0.0556 | Regression_pt: 1.4784 | Regression_sin_phi: 0.0780 | LR: 1.27e-04 | DataLoad Time: 0.0015s | Model Forward Time: 0.5663s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.1774 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.2 | uss_MB: 1610.4 | child_rss_MB: 22494.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 18.3 +rank0 [2026-11-14 07:03:30,228] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 07:03:30,228] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:33,068] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 07:04:33,068] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:59,265] INFO : Step 48000/500000 rank0 | Train Loss: 6.5567 | Train Loss Unweighted: 3.3682 | Train Loss Components: Classification: 0.0277 | Classification_binary: 3.0666 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1214 | Regression_eta: 0.0127 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3107 | Classification_binary: 3.0666 | Regression_cos_phi: 0.0981 | Regression_energy: 1.2658 | Regression_eta: 0.1000 | Regression_pt: 1.6314 | Regression_sin_phi: 0.0842 | LR: 1.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.6064s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.2083 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.3 | uss_MB: 1610.5 | child_rss_MB: 22494.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:04:59,267] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:04:59,337] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 07:05:36,210] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 07:05:36,210] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:38,771] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 07:06:38,772] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:51,076] INFO : Step 49000/500000 rank0 | Train Loss: 7.3159 | Train Loss Unweighted: 3.5160 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.1603 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1511 | Regression_eta: 0.0050 | Regression_pt: 0.1527 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3365 | Classification_binary: 3.1603 | Regression_cos_phi: 0.0677 | Regression_energy: 1.5751 | Regression_eta: 0.0394 | Regression_pt: 2.0196 | Regression_sin_phi: 0.1173 | LR: 1.36e-04 | DataLoad Time: 0.0021s | Model Forward Time: 0.5972s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0029 | max_backward: 0.0408 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1996 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.4 | uss_MB: 1610.5 | child_rss_MB: 22492.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:07:42,143] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 07:07:42,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:08:43,863] INFO : Step 50000/500000 rank0 | Train Loss: 6.4145 | Train Loss Unweighted: 3.4731 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.1958 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1181 | Regression_eta: 0.0068 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3097 | Classification_binary: 3.1958 | Regression_cos_phi: 0.0666 | Regression_energy: 1.2309 | Regression_eta: 0.0537 | Regression_pt: 1.4943 | Regression_sin_phi: 0.0635 | LR: 1.40e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.6453s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0030 | max_backward: 0.0449 | max_optimizer_step: 0.0711 | max_tensorboard_logging: 0.2431 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2751.5 | uss_MB: 1607.7 | child_rss_MB: 22494.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:08:43,866] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:08:43,940] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 07:08:43,941] INFO : Running validation on rank0 +rank0 [2026-11-14 07:08:43,941] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:08:43,941] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:08:44,097] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 07:08:44,218] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:08:44,333] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:08:44,403] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:08:44,470] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:08:44,536] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:08:44,604] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:08:44,671] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:08:44,768] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:08:44,884] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:08:44,999] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:08:45,068] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:08:45,136] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:08:45,202] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:08:45,270] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:08:45,338] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:08:45,456] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:08:45,462] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:08:45,464] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:08:45,467] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:08:45,469] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:08:45,471] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:08:45,474] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:08:45,476] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:08:45,478] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:08:45,481] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:08:45,483] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:08:45,485] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:08:45,487] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:08:45,490] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:08:45,492] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:08:45,494] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:08:47,078] INFO : VALIDATION | Step=50000/500000 | Train Loss=6.4145 | Valid Loss=8.1470 | Train Loss Unweighted=3.4731 | Valid Loss Unweighted=3.7909 | Stale=0 | ETA=851.5m +rank0 [2026-11-14 07:08:47,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:08:47,701] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:08:47,726] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:47,781] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:08:48,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:08:48,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:08:48,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:08:49,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:08:49,360] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:08:49,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:08:49,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:08:50,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:08:50,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:08:50,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:08:50,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:08:51,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:08:51,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:08:51,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:08:51,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:08:52,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:08:52,247] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:08:52,259] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:52,860] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:53,262] INFO : Plotted number of elements +rank0 [2026-11-14 07:08:54,617] INFO : Plotted jets +rank0 [2026-11-14 07:08:55,625] INFO : Plotted jet ratio +rank0 [2026-11-14 07:08:58,131] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 07:08:58,131] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:10:01,338] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 07:10:01,338] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:10:49,244] INFO : Step 51000/500000 rank0 | Train Loss: 7.8427 | Train Loss Unweighted: 3.7592 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.3727 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1565 | Regression_eta: 0.0195 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3121 | Classification_binary: 3.3727 | Regression_cos_phi: 0.1052 | Regression_energy: 1.6316 | Regression_eta: 0.1539 | Regression_pt: 2.1531 | Regression_sin_phi: 0.1141 | LR: 1.44e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.6096s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0028 | max_backward: 0.0404 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.2070 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2884.0 | uss_MB: 1740.2 | child_rss_MB: 22496.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:11:05,025] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 07:11:05,025] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:12:07,758] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 07:12:07,758] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:12:41,381] INFO : Step 52000/500000 rank0 | Train Loss: 6.9050 | Train Loss Unweighted: 3.4499 | Train Loss Components: Classification: 0.0351 | Classification_binary: 3.1232 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1309 | Regression_eta: 0.0106 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3928 | Classification_binary: 3.1232 | Regression_cos_phi: 0.1231 | Regression_energy: 1.3642 | Regression_eta: 0.0838 | Regression_pt: 1.7135 | Regression_sin_phi: 0.1044 | LR: 1.49e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5517s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0406 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.1488 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2884.0 | uss_MB: 1740.2 | child_rss_MB: 22498.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:12:41,383] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:12:41,457] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:13:11,041] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 07:13:11,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:14:13,717] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 07:14:13,717] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:14:33,286] INFO : Step 53000/500000 rank0 | Train Loss: 6.4875 | Train Loss Unweighted: 3.2492 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.9457 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1271 | Regression_eta: 0.0085 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2515 | Classification_binary: 2.9457 | Regression_cos_phi: 0.0680 | Regression_energy: 1.3253 | Regression_eta: 0.0668 | Regression_pt: 1.7692 | Regression_sin_phi: 0.0610 | LR: 1.53e-04 | DataLoad Time: 0.0025s | Model Forward Time: 0.5946s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0416 | max_optimizer_step: 0.0705 | max_tensorboard_logging: 0.1885 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2884.1 | uss_MB: 1740.2 | child_rss_MB: 22499.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:15:17,490] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 07:15:17,491] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:22,983] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 07:16:22,984] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:28,967] INFO : Step 54000/500000 rank0 | Train Loss: 6.7482 | Train Loss Unweighted: 3.4032 | Train Loss Components: Classification: 0.0321 | Classification_binary: 3.0859 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1302 | Regression_eta: 0.0130 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3600 | Classification_binary: 3.0859 | Regression_cos_phi: 0.0786 | Regression_energy: 1.3570 | Regression_eta: 0.1024 | Regression_pt: 1.6657 | Regression_sin_phi: 0.0986 | LR: 1.58e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.6236s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0416 | max_optimizer_step: 0.0713 | max_tensorboard_logging: 0.2213 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2777.6 | uss_MB: 1633.8 | child_rss_MB: 22497.5 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:16:28,969] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:16:29,040] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:17:26,859] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 07:17:26,859] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:18:22,050] INFO : Step 55000/500000 rank0 | Train Loss: 6.3495 | Train Loss Unweighted: 3.2879 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.0004 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1185 | Regression_eta: 0.0064 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3123 | Classification_binary: 3.0004 | Regression_cos_phi: 0.0799 | Regression_energy: 1.2350 | Regression_eta: 0.0507 | Regression_pt: 1.5934 | Regression_sin_phi: 0.0778 | LR: 1.62e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.5676s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0030 | max_backward: 0.0406 | max_optimizer_step: 0.0683 | max_tensorboard_logging: 0.1756 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2781.5 | uss_MB: 1637.7 | child_rss_MB: 22496.7 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:18:30,547] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 07:18:30,547] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:19:33,636] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 07:19:33,636] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:20:14,689] INFO : Step 56000/500000 rank0 | Train Loss: 6.3309 | Train Loss Unweighted: 3.1427 | Train Loss Components: Classification: 0.0263 | Classification_binary: 2.8434 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1283 | Regression_eta: 0.0064 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2945 | Classification_binary: 2.8434 | Regression_cos_phi: 0.0729 | Regression_energy: 1.3378 | Regression_eta: 0.0507 | Regression_pt: 1.6822 | Regression_sin_phi: 0.0494 | LR: 1.67e-04 | DataLoad Time: 0.0017s | Model Forward Time: 0.5696s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0029 | max_backward: 0.0420 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1786 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2782.0 | uss_MB: 1638.1 | child_rss_MB: 22497.8 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:20:14,692] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:20:14,769] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:20:37,026] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 07:20:37,026] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:21:39,829] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 07:21:39,830] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:22:07,122] INFO : Step 57000/500000 rank0 | Train Loss: 6.1664 | Train Loss Unweighted: 3.1832 | Train Loss Components: Classification: 0.0249 | Classification_binary: 2.8996 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1159 | Regression_eta: 0.0132 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2785 | Classification_binary: 2.8996 | Regression_cos_phi: 0.1001 | Regression_energy: 1.2082 | Regression_eta: 0.1037 | Regression_pt: 1.4821 | Regression_sin_phi: 0.0942 | LR: 1.72e-04 | DataLoad Time: 0.0021s | Model Forward Time: 0.5878s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0030 | max_backward: 0.0417 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1951 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2782.1 | uss_MB: 1638.2 | child_rss_MB: 22491.3 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:22:43,591] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 07:22:43,591] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:23:47,213] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 07:23:47,214] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:24:00,379] INFO : Step 58000/500000 rank0 | Train Loss: 6.3335 | Train Loss Unweighted: 3.4289 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.1532 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1110 | Regression_eta: 0.0110 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3035 | Classification_binary: 3.1532 | Regression_cos_phi: 0.0899 | Regression_energy: 1.1569 | Regression_eta: 0.0867 | Regression_pt: 1.4266 | Regression_sin_phi: 0.1167 | LR: 1.76e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.5348s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0027 | max_backward: 0.0407 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1379 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2777.1 | uss_MB: 1633.3 | child_rss_MB: 22491.2 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:24:00,381] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:24:00,460] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:24:50,830] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 07:24:50,830] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:25:53,150] INFO : Step 59000/500000 rank0 | Train Loss: 5.7660 | Train Loss Unweighted: 3.1763 | Train Loss Components: Classification: 0.0251 | Classification_binary: 2.9333 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0975 | Regression_eta: 0.0064 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2807 | Classification_binary: 2.9333 | Regression_cos_phi: 0.0672 | Regression_energy: 1.0168 | Regression_eta: 0.0505 | Regression_pt: 1.3674 | Regression_sin_phi: 0.0501 | LR: 1.81e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5993s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0413 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.2004 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2781.5 | uss_MB: 1637.7 | child_rss_MB: 22491.1 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:25:54,275] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 07:25:54,275] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:26:56,506] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 07:26:56,506] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:27:45,342] INFO : Step 60000/500000 rank0 | Train Loss: 6.4657 | Train Loss Unweighted: 3.3919 | Train Loss Components: Classification: 0.0221 | Classification_binary: 3.1040 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1219 | Regression_eta: 0.0058 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2478 | Classification_binary: 3.1040 | Regression_cos_phi: 0.0930 | Regression_energy: 1.2707 | Regression_eta: 0.0454 | Regression_pt: 1.6436 | Regression_sin_phi: 0.0612 | LR: 1.86e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.6041s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0711 | max_tensorboard_logging: 0.2022 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.7 | uss_MB: 1637.9 | child_rss_MB: 22490.7 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:27:45,344] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:27:45,426] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:27:45,428] INFO : Running validation on rank0 +rank0 [2026-11-14 07:27:45,428] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:27:45,428] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:27:45,591] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 07:27:45,712] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:27:45,828] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:27:45,897] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:27:45,965] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:27:46,034] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:27:46,104] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:27:46,173] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:27:46,275] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:27:46,392] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:27:46,506] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:27:46,574] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:27:46,643] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:27:46,710] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:27:46,778] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:27:46,847] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:27:46,958] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:27:46,964] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:27:46,966] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:27:46,969] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:27:46,972] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:27:46,974] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:27:46,976] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:27:46,978] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:27:46,980] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:27:46,983] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:27:46,985] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:27:46,987] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:27:46,990] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:27:46,992] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:27:46,994] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:27:46,997] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:27:48,526] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.4657 | Valid Loss=8.4191 | Train Loss Unweighted=3.3919 | Valid Loss Unweighted=3.8145 | Stale=1 | ETA=833.3m +rank0 [2026-11-14 07:27:48,527] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:27:49,275] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:27:49,288] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:49,345] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:27:49,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:27:50,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:27:50,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:27:50,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:27:50,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:27:51,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:27:51,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:27:51,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:27:51,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:27:52,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:27:52,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:27:52,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:27:53,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:27:53,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:27:53,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:27:53,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:27:53,813] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:27:53,832] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:54,433] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:54,839] INFO : Plotted number of elements +rank0 [2026-11-14 07:27:56,220] INFO : Plotted jets +rank0 [2026-11-14 07:27:57,227] INFO : Plotted jet ratio +rank0 [2026-11-14 07:28:12,493] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 07:28:12,493] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:14,937] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 07:29:14,937] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:49,667] INFO : Step 61000/500000 rank0 | Train Loss: 5.7818 | Train Loss Unweighted: 3.0853 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.8315 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1037 | Regression_eta: 0.0063 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2399 | Classification_binary: 2.8315 | Regression_cos_phi: 0.0738 | Regression_energy: 1.0807 | Regression_eta: 0.0497 | Regression_pt: 1.3858 | Regression_sin_phi: 0.1203 | LR: 1.91e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.5516s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0029 | max_backward: 0.0410 | max_optimizer_step: 0.0715 | max_tensorboard_logging: 0.1395 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2911.4 | uss_MB: 1767.6 | child_rss_MB: 22491.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 26.3 +rank0 [2026-11-14 07:30:18,200] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 07:30:18,200] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:31:20,430] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 07:31:20,430] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:31:41,362] INFO : Step 62000/500000 rank0 | Train Loss: 7.1917 | Train Loss Unweighted: 3.7193 | Train Loss Components: Classification: 0.0252 | Classification_binary: 3.3948 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1383 | Regression_eta: 0.0063 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2822 | Classification_binary: 3.3948 | Regression_cos_phi: 0.0760 | Regression_energy: 1.4416 | Regression_eta: 0.0494 | Regression_pt: 1.9033 | Regression_sin_phi: 0.0445 | LR: 1.95e-04 | DataLoad Time: 0.0020s | Model Forward Time: 0.6371s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0396 | max_loss: 0.0031 | max_backward: 0.0409 | max_optimizer_step: 0.0719 | max_tensorboard_logging: 0.2221 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2803.5 | uss_MB: 1659.7 | child_rss_MB: 22489.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:31:41,365] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:31:41,440] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:32:24,668] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 07:32:24,668] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:33:27,858] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 07:33:27,858] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:33:34,690] INFO : Step 63000/500000 rank0 | Train Loss: 5.8449 | Train Loss Unweighted: 3.3032 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.0623 | Regression_cos_phi: 0.0092 | Regression_energy: 0.0938 | Regression_eta: 0.0088 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3030 | Classification_binary: 3.0623 | Regression_cos_phi: 0.1031 | Regression_energy: 0.9782 | Regression_eta: 0.0693 | Regression_pt: 1.2332 | Regression_sin_phi: 0.0958 | LR: 2.00e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.6138s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.2008 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2806.9 | uss_MB: 1663.1 | child_rss_MB: 22490.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:34:31,349] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 07:34:31,349] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:35:27,229] INFO : Step 64000/500000 rank0 | Train Loss: 5.5282 | Train Loss Unweighted: 3.0770 | Train Loss Components: Classification: 0.0248 | Classification_binary: 2.8426 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0936 | Regression_eta: 0.0153 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2775 | Classification_binary: 2.8426 | Regression_cos_phi: 0.0545 | Regression_energy: 0.9757 | Regression_eta: 0.1202 | Regression_pt: 1.2061 | Regression_sin_phi: 0.0516 | LR: 2.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 0.5679s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0030 | max_backward: 0.0419 | max_optimizer_step: 0.0705 | max_tensorboard_logging: 0.1673 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.9 | uss_MB: 1663.1 | child_rss_MB: 22489.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:35:27,232] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:35:27,314] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:35:34,924] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 07:35:34,924] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:36:38,004] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 07:36:38,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:37:20,826] INFO : Step 65000/500000 rank0 | Train Loss: 6.2154 | Train Loss Unweighted: 3.2558 | Train Loss Components: Classification: 0.0284 | Classification_binary: 2.9784 | Regression_cos_phi: 0.0047 | Regression_energy: 0.1181 | Regression_eta: 0.0040 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3185 | Classification_binary: 2.9784 | Regression_cos_phi: 0.0532 | Regression_energy: 1.2314 | Regression_eta: 0.0316 | Regression_pt: 1.5485 | Regression_sin_phi: 0.0539 | LR: 2.10e-04 | DataLoad Time: 0.0022s | Model Forward Time: 0.5948s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0030 | max_backward: 0.0448 | max_optimizer_step: 0.0730 | max_tensorboard_logging: 0.1813 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2807.0 | uss_MB: 1663.2 | child_rss_MB: 22491.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:37:43,375] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 07:37:43,375] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:38:48,408] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 07:38:48,408] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:39:16,077] INFO : Step 66000/500000 rank0 | Train Loss: 6.3929 | Train Loss Unweighted: 3.1117 | Train Loss Components: Classification: 0.0350 | Classification_binary: 2.8030 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1250 | Regression_eta: 0.0079 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3925 | Classification_binary: 2.8030 | Regression_cos_phi: 0.0713 | Regression_energy: 1.3026 | Regression_eta: 0.0625 | Regression_pt: 1.6778 | Regression_sin_phi: 0.0832 | LR: 2.15e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.5940s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0402 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1992 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.0 | uss_MB: 1659.2 | child_rss_MB: 22493.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:39:16,079] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:39:16,155] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:39:51,590] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 07:39:51,590] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:40:54,351] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 07:40:54,351] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:41:08,495] INFO : Step 67000/500000 rank0 | Train Loss: 5.2425 | Train Loss Unweighted: 2.8737 | Train Loss Components: Classification: 0.0246 | Classification_binary: 2.6501 | Regression_cos_phi: 0.0071 | Regression_energy: 0.0875 | Regression_eta: 0.0063 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2758 | Classification_binary: 2.6501 | Regression_cos_phi: 0.0797 | Regression_energy: 0.9123 | Regression_eta: 0.0494 | Regression_pt: 1.1745 | Regression_sin_phi: 0.1007 | LR: 2.20e-04 | DataLoad Time: 0.0013s | Model Forward Time: 0.5675s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0412 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1712 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.3 | uss_MB: 1662.5 | child_rss_MB: 22493.6 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:41:57,982] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 07:41:57,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:43:01,840] INFO : Step 68000/500000 rank0 | Train Loss: 5.8962 | Train Loss Unweighted: 3.2217 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.9707 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1025 | Regression_eta: 0.0051 | Regression_pt: 0.1053 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2767 | Classification_binary: 2.9707 | Regression_cos_phi: 0.0747 | Regression_energy: 1.0688 | Regression_eta: 0.0402 | Regression_pt: 1.3935 | Regression_sin_phi: 0.0716 | LR: 2.25e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.5722s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0367 | max_loss: 0.0030 | max_backward: 0.0409 | max_optimizer_step: 0.0735 | max_tensorboard_logging: 0.1651 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.4 | uss_MB: 1662.6 | child_rss_MB: 22491.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:43:01,842] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:43:01,922] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:43:02,143] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 07:43:02,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:44:05,189] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 07:44:05,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:44:54,528] INFO : Step 69000/500000 rank0 | Train Loss: 5.4393 | Train Loss Unweighted: 2.8107 | Train Loss Components: Classification: 0.0274 | Classification_binary: 2.5638 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1041 | Regression_eta: 0.0039 | Regression_pt: 0.1012 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3072 | Classification_binary: 2.5638 | Regression_cos_phi: 0.0588 | Regression_energy: 1.0852 | Regression_eta: 0.0308 | Regression_pt: 1.3383 | Regression_sin_phi: 0.0553 | LR: 2.30e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.5819s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0407 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.1865 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.5 | uss_MB: 1662.7 | child_rss_MB: 22492.5 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:45:08,583] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 07:45:08,583] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:46:11,806] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 07:46:11,806] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:46:47,602] INFO : Step 70000/500000 rank0 | Train Loss: 4.8632 | Train Loss Unweighted: 2.7565 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.5584 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0802 | Regression_eta: 0.0041 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2517 | Classification_binary: 2.5584 | Regression_cos_phi: 0.0467 | Regression_energy: 0.8364 | Regression_eta: 0.0327 | Regression_pt: 1.0719 | Regression_sin_phi: 0.0654 | LR: 2.35e-04 | DataLoad Time: 0.0022s | Model Forward Time: 0.6094s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0030 | max_backward: 0.0415 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.2087 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.0 | uss_MB: 1659.2 | child_rss_MB: 22493.8 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:46:47,605] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:46:47,684] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:46:47,685] INFO : Running validation on rank0 +rank0 [2026-11-14 07:46:47,685] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:46:47,685] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:46:47,843] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 07:46:47,964] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:46:48,082] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:46:48,153] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:46:48,221] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:46:48,291] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:46:48,362] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:46:48,433] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:46:48,538] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:46:48,655] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:46:48,771] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:46:48,839] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:46:48,906] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:46:48,974] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:46:49,045] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:46:49,115] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:46:49,225] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:46:49,231] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:46:49,233] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:46:49,236] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:46:49,238] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:46:49,241] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:46:49,243] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:46:49,246] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:46:49,248] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:46:49,250] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:46:49,253] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:46:49,255] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:46:49,258] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:46:49,260] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:46:49,262] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:46:49,265] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:46:50,799] INFO : VALIDATION | Step=70000/500000 | Train Loss=4.8632 | Valid Loss=8.7554 | Train Loss Unweighted=2.7565 | Valid Loss Unweighted=3.9366 | Stale=2 | ETA=815.0m +rank0 [2026-11-14 07:46:50,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:46:51,313] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:46:51,326] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:46:51,382] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:46:51,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:46:52,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:46:52,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:46:52,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:46:52,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:46:53,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:46:53,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:46:53,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:46:53,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:46:54,258] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:46:54,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:46:54,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:46:55,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:46:55,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:46:55,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:46:55,770] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:46:55,830] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:46:55,852] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:46:56,453] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:46:56,848] INFO : Plotted number of elements +rank0 [2026-11-14 07:46:58,214] INFO : Plotted jets +rank0 [2026-11-14 07:46:59,218] INFO : Plotted jet ratio +rank0 [2026-11-14 07:47:27,722] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 07:47:27,722] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:48:31,245] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 07:48:31,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:48:52,796] INFO : Step 71000/500000 rank0 | Train Loss: 5.1706 | Train Loss Unweighted: 2.8523 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.6305 | Regression_cos_phi: 0.0080 | Regression_energy: 0.0835 | Regression_eta: 0.0139 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2409 | Classification_binary: 2.6305 | Regression_cos_phi: 0.0891 | Regression_energy: 0.8709 | Regression_eta: 0.1093 | Regression_pt: 1.1162 | Regression_sin_phi: 0.1136 | LR: 2.40e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.5760s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0399 | max_optimizer_step: 0.0704 | max_tensorboard_logging: 0.1725 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2936.9 | uss_MB: 1793.1 | child_rss_MB: 22495.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:49:35,048] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 07:49:35,048] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:50:38,082] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 07:50:38,082] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:50:45,814] INFO : Step 72000/500000 rank0 | Train Loss: 6.3488 | Train Loss Unweighted: 3.4243 | Train Loss Components: Classification: 0.0264 | Classification_binary: 3.1491 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1127 | Regression_eta: 0.0064 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2958 | Classification_binary: 3.1491 | Regression_cos_phi: 0.1035 | Regression_energy: 1.1753 | Regression_eta: 0.0503 | Regression_pt: 1.4885 | Regression_sin_phi: 0.0864 | LR: 2.45e-04 | DataLoad Time: 0.0022s | Model Forward Time: 0.6069s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0380 | max_loss: 0.0030 | max_backward: 0.0426 | max_optimizer_step: 0.0718 | max_tensorboard_logging: 0.2036 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2936.9 | uss_MB: 1793.1 | child_rss_MB: 22496.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:50:45,816] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 07:50:45,892] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:51:41,509] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 07:51:41,509] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:52:38,493] INFO : Step 73000/500000 rank0 | Train Loss: 4.7876 | Train Loss Unweighted: 2.8118 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.6239 | Regression_cos_phi: 0.0108 | Regression_energy: 0.0665 | Regression_eta: 0.0076 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2373 | Classification_binary: 2.6239 | Regression_cos_phi: 0.1208 | Regression_energy: 0.6932 | Regression_eta: 0.0602 | Regression_pt: 0.9128 | Regression_sin_phi: 0.1394 | LR: 2.50e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.5690s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0037 | max_backward: 0.0409 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1599 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2936.9 | uss_MB: 1793.2 | child_rss_MB: 22497.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:52:45,241] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 07:52:45,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:53:46,894] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 07:53:46,894] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:54:29,414] INFO : Step 74000/500000 rank0 | Train Loss: 4.6294 | Train Loss Unweighted: 2.6329 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4446 | Regression_cos_phi: 0.0041 | Regression_energy: 0.0772 | Regression_eta: 0.0050 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2278 | Classification_binary: 2.4446 | Regression_cos_phi: 0.0457 | Regression_energy: 0.8045 | Regression_eta: 0.0395 | Regression_pt: 1.0055 | Regression_sin_phi: 0.0618 | LR: 2.55e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.6062s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0405 | max_optimizer_step: 0.0715 | max_tensorboard_logging: 0.1984 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2827.1 | uss_MB: 1683.3 | child_rss_MB: 22495.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:54:29,416] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 07:54:29,494] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:54:50,484] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 07:54:50,485] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:55:53,177] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 07:55:53,177] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:56:21,917] INFO : Step 75000/500000 rank0 | Train Loss: 5.2123 | Train Loss Unweighted: 2.9845 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.7712 | Regression_cos_phi: 0.0082 | Regression_energy: 0.0832 | Regression_eta: 0.0113 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2645 | Classification_binary: 2.7712 | Regression_cos_phi: 0.0923 | Regression_energy: 0.8676 | Regression_eta: 0.0891 | Regression_pt: 1.0237 | Regression_sin_phi: 0.1039 | LR: 2.60e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5381s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0680 | max_tensorboard_logging: 0.1549 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2829.8 | uss_MB: 1686.0 | child_rss_MB: 22496.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:56:56,154] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 07:56:56,154] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:57:58,864] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 07:57:58,864] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:58:13,849] INFO : Step 76000/500000 rank0 | Train Loss: 5.0237 | Train Loss Unweighted: 2.8620 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.6563 | Regression_cos_phi: 0.0074 | Regression_energy: 0.0807 | Regression_eta: 0.0114 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2067 | Classification_binary: 2.6563 | Regression_cos_phi: 0.0827 | Regression_energy: 0.8415 | Regression_eta: 0.0896 | Regression_pt: 1.0826 | Regression_sin_phi: 0.0643 | LR: 2.65e-04 | DataLoad Time: 0.0019s | Model Forward Time: 0.6040s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.2093 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2829.8 | uss_MB: 1686.1 | child_rss_MB: 22494.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:58:13,851] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 07:58:13,922] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:59:02,063] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 07:59:02,063] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:05,774] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 08:00:05,775] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:06,991] INFO : Step 77000/500000 rank0 | Train Loss: 4.7542 | Train Loss Unweighted: 2.7273 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.5323 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0758 | Regression_eta: 0.0134 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2298 | Classification_binary: 2.5323 | Regression_cos_phi: 0.0651 | Regression_energy: 0.7905 | Regression_eta: 0.1055 | Regression_pt: 0.9415 | Regression_sin_phi: 0.0896 | LR: 2.70e-04 | DataLoad Time: 0.0003s | Model Forward Time: 0.6037s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0379 | max_loss: 0.0032 | max_backward: 0.0430 | max_optimizer_step: 0.0734 | max_tensorboard_logging: 0.1861 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.0 | uss_MB: 1686.3 | child_rss_MB: 22495.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:01:11,545] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 08:01:11,545] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:01,287] INFO : Step 78000/500000 rank0 | Train Loss: 4.5251 | Train Loss Unweighted: 2.7048 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5321 | Regression_cos_phi: 0.0090 | Regression_energy: 0.0700 | Regression_eta: 0.0051 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2022 | Classification_binary: 2.5321 | Regression_cos_phi: 0.1004 | Regression_energy: 0.7297 | Regression_eta: 0.0402 | Regression_pt: 0.8598 | Regression_sin_phi: 0.0608 | LR: 2.75e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.6044s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1896 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.0 | uss_MB: 1686.2 | child_rss_MB: 22496.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:02:01,289] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 08:02:01,367] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 08:02:14,628] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 08:02:14,628] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:03:17,382] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 08:03:17,383] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:03:53,226] INFO : Step 79000/500000 rank0 | Train Loss: 4.3472 | Train Loss Unweighted: 2.4582 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.2808 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0721 | Regression_eta: 0.0044 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2105 | Classification_binary: 2.2808 | Regression_cos_phi: 0.0557 | Regression_energy: 0.7512 | Regression_eta: 0.0344 | Regression_pt: 0.9783 | Regression_sin_phi: 0.0363 | LR: 2.80e-04 | DataLoad Time: 0.0015s | Model Forward Time: 0.6502s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0697 | max_tensorboard_logging: 0.2391 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2829.7 | uss_MB: 1685.9 | child_rss_MB: 22497.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 08:04:20,044] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 08:04:20,044] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:22,987] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 08:05:22,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:45,997] INFO : Step 80000/500000 rank0 | Train Loss: 4.4951 | Train Loss Unweighted: 2.6166 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.4391 | Regression_cos_phi: 0.0063 | Regression_energy: 0.0700 | Regression_eta: 0.0044 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2470 | Classification_binary: 2.4391 | Regression_cos_phi: 0.0703 | Regression_energy: 0.7302 | Regression_eta: 0.0348 | Regression_pt: 0.9024 | Regression_sin_phi: 0.0714 | LR: 2.85e-04 | DataLoad Time: 0.0016s | Model Forward Time: 0.6685s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0404 | max_optimizer_step: 0.0705 | max_tensorboard_logging: 0.2682 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2830.4 | uss_MB: 1686.7 | child_rss_MB: 22496.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 08:05:45,999] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 08:05:46,078] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 08:05:46,080] INFO : Running validation on rank0 +rank0 [2026-11-14 08:05:46,080] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 08:05:46,080] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 08:05:46,236] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 08:05:46,357] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 08:05:46,476] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 08:05:46,547] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 08:05:46,615] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 08:05:46,682] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 08:05:46,751] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 08:05:46,819] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 08:05:46,923] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 08:05:47,040] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 08:05:47,159] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 08:05:47,228] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 08:05:47,295] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 08:05:47,362] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 08:05:47,430] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 08:05:47,497] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 08:05:47,607] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 08:05:47,613] INFO : plotted pt_target histogram +rank0 [2026-11-14 08:05:47,615] INFO : plotted pt_pred histogram +rank0 [2026-11-14 08:05:47,617] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 08:05:47,619] INFO : plotted eta_target histogram +rank0 [2026-11-14 08:05:47,622] INFO : plotted eta_pred histogram +rank0 [2026-11-14 08:05:47,624] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 08:05:47,626] INFO : plotted sphi_target histogram +rank0 [2026-11-14 08:05:47,628] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 08:05:47,631] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 08:05:47,633] INFO : plotted cphi_target histogram +rank0 [2026-11-14 08:05:47,635] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 08:05:47,637] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 08:05:47,640] INFO : plotted energy_target histogram +rank0 [2026-11-14 08:05:47,642] INFO : plotted energy_pred histogram +rank0 [2026-11-14 08:05:47,644] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 08:05:49,174] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.4951 | Valid Loss=9.6425 | Train Loss Unweighted=2.6166 | Valid Loss Unweighted=4.3894 | Stale=3 | ETA=796.2m +rank0 [2026-11-14 08:05:49,174] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 08:05:49,637] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 08:05:49,651] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 08:05:49,705] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 08:05:50,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 08:05:50,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 08:05:50,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 08:05:51,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 08:05:51,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 08:05:51,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 08:05:51,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 08:05:52,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 08:05:52,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 08:05:52,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 08:05:52,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 08:05:53,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 08:05:53,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 08:05:53,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 08:05:53,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 08:05:54,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 08:05:54,198] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 08:05:54,207] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:05:54,785] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:05:55,185] INFO : Plotted number of elements +rank0 [2026-11-14 08:05:56,528] INFO : Plotted jets +rank0 [2026-11-14 08:05:57,508] INFO : Plotted jet ratio +rank0 [2026-11-14 08:05:57,986] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 08:05:57,986] INFO : Training completed. Total time on device 0: 151.796min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..78f87a5431b06dd8f0e98d1bc2534125e75f6736 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/train.log.0 @@ -0,0 +1,1351 @@ +rank0 [2026-11-14 05:34:06,956] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 05:34:06,956] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 05:34:06,958] INFO : Instantiating model +rank0 [2026-11-14 05:34:06,980] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 05:34:06,980] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 05:34:06,982] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 05:34:06,982] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 05:34:06,982] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 05:34:06,982] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 05:34:06,982] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,985] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 05:34:06,985] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,988] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 05:34:06,988] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,990] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 05:34:06,990] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,993] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 05:34:06,993] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,996] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 05:34:06,996] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:06,999] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 05:34:06,999] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 05:34:06,999] INFO : Initializing output DNNs +rank0 [2026-11-14 05:34:06,999] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:07,003] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 05:34:07,010] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 05:34:07,010] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 05:34:07,010] INFO : jagged_attention enabled=True +rank0 [2026-11-14 05:34:07,010] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 05:34:07,010] INFO : task_queries enabled=True +rank0 [2026-11-14 05:34:07,010] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 05:34:07,010] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 05:34:07,010] INFO : classification_query parameters: 256 +rank0 [2026-11-14 05:34:07,010] INFO : regression_query parameters: 256 +rank0 [2026-11-14 05:34:07,010] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 05:34:07,010] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 05:34:07,010] INFO : output_mode=elementwise +rank0 [2026-11-14 05:34:07,010] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 05:34:07,010] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 05:34:07,010] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 05:34:07,011] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 05:34:07,011] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 05:34:07,011] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 05:34:07,011] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 05:34:07,011] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 05:34:07,012] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 05:34:07,012] INFO : MLPF __init__ done +rank0 [2026-11-14 05:34:07,012] INFO : Instantiated model +rank0 [2026-11-14 05:34:07,013] INFO : Moving model to device rank=0 +rank0 [2026-11-14 05:34:07,021] INFO : Moved model to device rank=0 +rank0 [2026-11-14 05:34:07,022] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 05:34:07,125] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 05:34:07,125] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 05:34:07,126] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 05:34:07,126] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 05:34:07,126] INFO : Total parameters: 3910930 +rank0 [2026-11-14 05:34:07,128] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 05:34:07,133] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046 +rank0 [2026-11-14 05:34:07,133] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046 +rank0 [2026-11-14 05:34:07,133] INFO : Getting dataloaders +rank0 [2026-11-14 05:34:07,133] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:34:09,414] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,503] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,554] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,596] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,658] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,706] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,750] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,823] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,868] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,906] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 05:34:09,906] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 05:34:09,906] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 05:34:09,906] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 05:34:09,906] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:34:09,914] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,923] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,936] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,946] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,965] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,974] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:09,993] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:10,005] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:10,018] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:10,027] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 05:34:10,027] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 05:34:10,027] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 05:34:10,075] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:34:10,234] INFO : Got dataloaders +rank0 [2026-11-14 05:34:10,234] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 05:34:10,235] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 05:34:10,235] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 05:34:10,241] INFO : Creating train_iterator +rank0 [2026-11-14 05:34:10,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:34:10,270] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:34:10,431] INFO : Created train_iterator +rank0 [2026-11-14 05:34:23,321] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.080610990524292, 'Classification': 0.6321391302347183, 'Regression_pt': 0.5351871705055237, 'Regression_eta': 0.8981421908736229, 'Regression_sin_phi': 0.653542291522026, 'Regression_cos_phi': 0.6319048485159874, 'Regression_energy': 0.6792134913802147}, weights={'Classification_binary': 1.0, 'Classification': 11.201032638549805, 'Regression_pt': 13.230157852172852, 'Regression_eta': 7.88361930847168, 'Regression_sin_phi': 10.83420467376709, 'Regression_cos_phi': 11.205184936523438, 'Regression_energy': 10.424720764160156} +rank0 [2026-11-14 05:35:15,762] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 05:35:15,762] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:36:05,181] INFO : Step 1000/500000 rank0 | Train Loss: 14.5445 | Train Loss Unweighted: 5.3518 | Train Loss Components: Classification: 0.0596 | Classification_binary: 4.4874 | Regression_cos_phi: 0.0198 | Regression_energy: 0.3652 | Regression_eta: 0.0245 | Regression_pt: 0.3693 | Regression_sin_phi: 0.0259 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.6678 | Classification_binary: 4.4874 | Regression_cos_phi: 0.2215 | Regression_energy: 3.8075 | Regression_eta: 0.1933 | Regression_pt: 4.8862 | Regression_sin_phi: 0.2807 | LR: 2.01e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.6164s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0029 | max_backward: 0.0404 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.2301 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2564.5 | uss_MB: 1405.3 | child_rss_MB: 22472.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:36:19,136] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 05:36:19,136] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:21,686] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 05:37:21,687] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:37:57,018] INFO : Step 2000/500000 rank0 | Train Loss: 12.5979 | Train Loss Unweighted: 5.1177 | Train Loss Components: Classification: 0.0403 | Classification_binary: 4.4129 | Regression_cos_phi: 0.0201 | Regression_energy: 0.2992 | Regression_eta: 0.0261 | Regression_pt: 0.3028 | Regression_sin_phi: 0.0163 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4518 | Classification_binary: 4.4129 | Regression_cos_phi: 0.2253 | Regression_energy: 3.1191 | Regression_eta: 0.2055 | Regression_pt: 4.0064 | Regression_sin_phi: 0.1769 | LR: 2.02e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.5733s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0028 | max_backward: 0.0422 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1864 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2566.1 | uss_MB: 1407.0 | child_rss_MB: 22475.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:37:57,020] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:38:24,803] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 05:38:24,803] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:27,775] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 05:39:27,776] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:39:49,291] INFO : Step 3000/500000 rank0 | Train Loss: 9.4411 | Train Loss Unweighted: 4.5168 | Train Loss Components: Classification: 0.0423 | Classification_binary: 4.0532 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1941 | Regression_eta: 0.0130 | Regression_pt: 0.1937 | Regression_sin_phi: 0.0101 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4741 | Classification_binary: 4.0532 | Regression_cos_phi: 0.1151 | Regression_energy: 2.0239 | Regression_eta: 0.1029 | Regression_pt: 2.5624 | Regression_sin_phi: 0.1095 | LR: 2.05e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.5798s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0383 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0671 | max_tensorboard_logging: 0.1965 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.0 | uss_MB: 1411.3 | child_rss_MB: 22473.7 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:40:30,507] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 05:40:30,507] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:32,985] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 05:41:32,985] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:41:40,864] INFO : Step 4000/500000 rank0 | Train Loss: 9.8202 | Train Loss Unweighted: 4.3056 | Train Loss Components: Classification: 0.0490 | Classification_binary: 3.7862 | Regression_cos_phi: 0.0174 | Regression_energy: 0.2147 | Regression_eta: 0.0130 | Regression_pt: 0.2122 | Regression_sin_phi: 0.0131 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.5491 | Classification_binary: 3.7862 | Regression_cos_phi: 0.1945 | Regression_energy: 2.2387 | Regression_eta: 0.1024 | Regression_pt: 2.8070 | Regression_sin_phi: 0.1424 | LR: 2.08e-05 | DataLoad Time: 0.0009s | Model Forward Time: 0.5806s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0409 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1961 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2570.0 | uss_MB: 1411.3 | child_rss_MB: 22474.9 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:41:40,866] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:42:36,201] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 05:42:36,201] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:43:33,156] INFO : Step 5000/500000 rank0 | Train Loss: 9.9054 | Train Loss Unweighted: 4.6113 | Train Loss Components: Classification: 0.0357 | Classification_binary: 4.1123 | Regression_cos_phi: 0.0157 | Regression_energy: 0.2079 | Regression_eta: 0.0167 | Regression_pt: 0.2095 | Regression_sin_phi: 0.0135 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3998 | Classification_binary: 4.1123 | Regression_cos_phi: 0.1762 | Regression_energy: 2.1670 | Regression_eta: 0.1320 | Regression_pt: 2.7715 | Regression_sin_phi: 0.1464 | LR: 2.13e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.5878s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1979 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2568.3 | uss_MB: 1409.6 | child_rss_MB: 22475.8 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:43:39,613] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 05:43:39,613] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:44:41,674] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 05:44:41,674] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:45:24,422] INFO : Step 6000/500000 rank0 | Train Loss: 10.6283 | Train Loss Unweighted: 4.5432 | Train Loss Components: Classification: 0.0393 | Classification_binary: 3.9723 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2514 | Regression_eta: 0.0116 | Regression_pt: 0.2460 | Regression_sin_phi: 0.0120 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4403 | Classification_binary: 3.9723 | Regression_cos_phi: 0.1186 | Regression_energy: 2.6212 | Regression_eta: 0.0912 | Regression_pt: 3.2550 | Regression_sin_phi: 0.1297 | LR: 2.19e-05 | DataLoad Time: 0.0014s | Model Forward Time: 0.6233s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0029 | max_backward: 0.0403 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.2328 | max_tensorboard_flush: 0.0005 | Memory: rss_MB: 2568.3 | uss_MB: 1409.6 | child_rss_MB: 22476.4 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:45:24,424] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:45:44,905] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 05:45:44,905] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:46:49,962] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 05:46:49,962] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:47:19,174] INFO : Step 7000/500000 rank0 | Train Loss: 10.2920 | Train Loss Unweighted: 4.7103 | Train Loss Components: Classification: 0.0412 | Classification_binary: 4.1889 | Regression_cos_phi: 0.0092 | Regression_energy: 0.2212 | Regression_eta: 0.0089 | Regression_pt: 0.2305 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4609 | Classification_binary: 4.1889 | Regression_cos_phi: 0.1026 | Regression_energy: 2.3064 | Regression_eta: 0.0700 | Regression_pt: 3.0492 | Regression_sin_phi: 0.1139 | LR: 2.26e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.6156s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0029 | max_backward: 0.0403 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.2092 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2568.9 | uss_MB: 1410.2 | child_rss_MB: 22479.0 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:47:53,183] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 05:47:53,183] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:48:56,150] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 05:48:56,151] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:49:11,298] INFO : Step 8000/500000 rank0 | Train Loss: 9.1124 | Train Loss Unweighted: 4.7340 | Train Loss Components: Classification: 0.0384 | Classification_binary: 4.3234 | Regression_cos_phi: 0.0091 | Regression_energy: 0.1744 | Regression_eta: 0.0064 | Regression_pt: 0.1729 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4307 | Classification_binary: 4.3234 | Regression_cos_phi: 0.1017 | Regression_energy: 1.8178 | Regression_eta: 0.0503 | Regression_pt: 2.2871 | Regression_sin_phi: 0.1014 | LR: 2.34e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.5642s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0367 | max_loss: 0.0029 | max_backward: 0.0411 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1798 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2568.9 | uss_MB: 1410.2 | child_rss_MB: 22475.3 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:49:11,300] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 05:49:11,376] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 05:49:58,172] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 05:49:58,172] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:00,519] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 05:51:00,519] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:51:01,915] INFO : Step 9000/500000 rank0 | Train Loss: 9.5025 | Train Loss Unweighted: 4.3462 | Train Loss Components: Classification: 0.0408 | Classification_binary: 3.8611 | Regression_cos_phi: 0.0132 | Regression_energy: 0.2007 | Regression_eta: 0.0129 | Regression_pt: 0.2032 | Regression_sin_phi: 0.0142 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4568 | Classification_binary: 3.8611 | Regression_cos_phi: 0.1482 | Regression_energy: 2.0924 | Regression_eta: 0.1017 | Regression_pt: 2.6885 | Regression_sin_phi: 0.1538 | LR: 2.43e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.5864s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1988 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2568.0 | uss_MB: 1409.3 | child_rss_MB: 22476.5 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 05:52:03,512] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 05:52:03,512] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:52:53,569] INFO : Step 10000/500000 rank0 | Train Loss: 9.6285 | Train Loss Unweighted: 4.3744 | Train Loss Components: Classification: 0.0381 | Classification_binary: 3.8801 | Regression_cos_phi: 0.0106 | Regression_energy: 0.2064 | Regression_eta: 0.0161 | Regression_pt: 0.2120 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4268 | Classification_binary: 3.8801 | Regression_cos_phi: 0.1191 | Regression_energy: 2.1517 | Regression_eta: 0.1272 | Regression_pt: 2.8053 | Regression_sin_phi: 0.1182 | LR: 2.52e-05 | DataLoad Time: 0.0019s | Model Forward Time: 0.5763s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0028 | max_backward: 0.0399 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1976 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2568.5 | uss_MB: 1409.8 | child_rss_MB: 22477.5 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 05:52:53,571] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 05:52:53,646] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 05:52:53,648] INFO : Running validation on rank0 +rank0 [2026-11-14 05:52:53,648] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 05:52:53,648] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 05:52:53,909] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 05:52:54,138] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 05:52:54,221] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 05:52:54,275] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 05:52:54,331] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 05:52:54,377] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 05:52:54,432] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 05:52:54,486] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 05:52:54,539] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 05:52:54,617] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 05:52:54,695] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 05:52:54,750] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 05:52:54,806] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 05:52:54,854] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 05:52:54,910] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 05:52:54,966] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 05:52:55,017] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 05:52:55,021] INFO : plotted pt_target histogram +rank0 [2026-11-14 05:52:55,023] INFO : plotted pt_pred histogram +rank0 [2026-11-14 05:52:55,025] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 05:52:55,028] INFO : plotted eta_target histogram +rank0 [2026-11-14 05:52:55,030] INFO : plotted eta_pred histogram +rank0 [2026-11-14 05:52:55,033] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 05:52:55,035] INFO : plotted sphi_target histogram +rank0 [2026-11-14 05:52:55,038] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 05:52:55,040] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 05:52:55,042] INFO : plotted cphi_target histogram +rank0 [2026-11-14 05:52:55,045] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 05:52:55,047] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 05:52:55,049] INFO : plotted energy_target histogram +rank0 [2026-11-14 05:52:55,052] INFO : plotted energy_pred histogram +rank0 [2026-11-14 05:52:55,054] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 05:52:56,631] INFO : VALIDATION | Step=10000/500000 | Train Loss=9.6285 | Valid Loss=9.2713 | Train Loss Unweighted=4.3744 | Valid Loss Unweighted=4.3800 | Stale=0 | ETA=919.9m +rank0 [2026-11-14 05:52:56,631] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 05:52:57,134] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 05:52:57,145] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 05:52:57,201] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 05:52:58,078] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 05:52:58,337] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 05:52:58,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 05:52:58,852] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 05:52:59,111] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 05:52:59,367] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 05:52:59,632] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 05:52:59,892] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 05:53:00,158] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 05:53:00,418] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 05:53:00,676] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 05:53:00,942] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 05:53:01,208] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 05:53:01,483] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 05:53:01,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 05:53:01,992] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 05:53:02,076] INFO : Time taken to make predictions on device 0 is: 0.08 min +rank0 [2026-11-14 05:53:02,085] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:53:02,673] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 05:53:03,794] INFO : Plotted number of elements +rank0 [2026-11-14 05:53:05,198] INFO : Plotted jets +rank0 [2026-11-14 05:53:06,204] INFO : Plotted jet ratio +rank0 [2026-11-14 05:53:19,623] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 05:53:19,623] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:22,550] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 05:54:22,551] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:54:59,042] INFO : Step 11000/500000 rank0 | Train Loss: 10.1912 | Train Loss Unweighted: 4.5144 | Train Loss Components: Classification: 0.0474 | Classification_binary: 3.9782 | Regression_cos_phi: 0.0167 | Regression_energy: 0.2158 | Regression_eta: 0.0167 | Regression_pt: 0.2161 | Regression_sin_phi: 0.0234 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.5314 | Classification_binary: 3.9782 | Regression_cos_phi: 0.1873 | Regression_energy: 2.2499 | Regression_eta: 0.1316 | Regression_pt: 2.8596 | Regression_sin_phi: 0.2532 | LR: 2.63e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5826s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0027 | max_backward: 0.0394 | max_optimizer_step: 0.0709 | max_tensorboard_logging: 0.1775 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2743.5 | uss_MB: 1599.4 | child_rss_MB: 22482.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 24.3 +rank0 [2026-11-14 05:55:25,821] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 05:55:25,821] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:28,094] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 05:56:28,095] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:56:50,624] INFO : Step 12000/500000 rank0 | Train Loss: 9.2129 | Train Loss Unweighted: 4.1807 | Train Loss Components: Classification: 0.0340 | Classification_binary: 3.7061 | Regression_cos_phi: 0.0159 | Regression_energy: 0.2010 | Regression_eta: 0.0133 | Regression_pt: 0.1954 | Regression_sin_phi: 0.0149 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3812 | Classification_binary: 3.7061 | Regression_cos_phi: 0.1779 | Regression_energy: 2.0955 | Regression_eta: 0.1048 | Regression_pt: 2.5858 | Regression_sin_phi: 0.1616 | LR: 2.75e-05 | DataLoad Time: 0.0011s | Model Forward Time: 0.5800s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0383 | max_loss: 0.0030 | max_backward: 0.0405 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.1849 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2744.1 | uss_MB: 1600.1 | child_rss_MB: 22484.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 05:56:50,626] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 05:56:50,704] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 05:57:31,172] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 05:57:31,172] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:33,609] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 05:58:33,609] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 05:58:42,192] INFO : Step 13000/500000 rank0 | Train Loss: 10.9813 | Train Loss Unweighted: 4.4905 | Train Loss Components: Classification: 0.0406 | Classification_binary: 3.8827 | Regression_cos_phi: 0.0105 | Regression_energy: 0.2655 | Regression_eta: 0.0134 | Regression_pt: 0.2685 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4544 | Classification_binary: 3.8827 | Regression_cos_phi: 0.1181 | Regression_energy: 2.7676 | Regression_eta: 0.1053 | Regression_pt: 3.5526 | Regression_sin_phi: 0.1005 | LR: 2.88e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.5990s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0357 | max_loss: 0.0036 | max_backward: 0.0409 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.2010 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2669.8 | uss_MB: 1525.8 | child_rss_MB: 22482.4 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 05:59:36,571] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 05:59:36,571] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:00:34,422] INFO : Step 14000/500000 rank0 | Train Loss: 8.9930 | Train Loss Unweighted: 4.3760 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.9433 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1805 | Regression_eta: 0.0115 | Regression_pt: 0.1898 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3349 | Classification_binary: 3.9433 | Regression_cos_phi: 0.1181 | Regression_energy: 1.8820 | Regression_eta: 0.0903 | Regression_pt: 2.5110 | Regression_sin_phi: 0.1133 | LR: 3.02e-05 | DataLoad Time: 0.0020s | Model Forward Time: 0.6538s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0401 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.2671 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2669.8 | uss_MB: 1525.8 | child_rss_MB: 22484.2 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:00:34,424] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:00:34,504] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 06:00:40,115] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 06:00:40,115] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:01:42,725] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 06:01:42,725] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:02:26,826] INFO : Step 15000/500000 rank0 | Train Loss: 9.9478 | Train Loss Unweighted: 4.2329 | Train Loss Components: Classification: 0.0436 | Classification_binary: 3.6984 | Regression_cos_phi: 0.0128 | Regression_energy: 0.2210 | Regression_eta: 0.0104 | Regression_pt: 0.2334 | Regression_sin_phi: 0.0133 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4879 | Classification_binary: 3.6984 | Regression_cos_phi: 0.1432 | Regression_energy: 2.3034 | Regression_eta: 0.0823 | Regression_pt: 3.0881 | Regression_sin_phi: 0.1446 | LR: 3.17e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.5951s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.2059 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2672.6 | uss_MB: 1528.6 | child_rss_MB: 22483.6 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:02:46,245] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 06:02:46,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:03:48,811] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 06:03:48,811] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:04:18,668] INFO : Step 16000/500000 rank0 | Train Loss: 8.6107 | Train Loss Unweighted: 4.0345 | Train Loss Components: Classification: 0.0365 | Classification_binary: 3.6061 | Regression_cos_phi: 0.0140 | Regression_energy: 0.1788 | Regression_eta: 0.0064 | Regression_pt: 0.1822 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4091 | Classification_binary: 3.6061 | Regression_cos_phi: 0.1569 | Regression_energy: 1.8643 | Regression_eta: 0.0506 | Regression_pt: 2.4107 | Regression_sin_phi: 0.1129 | LR: 3.33e-05 | DataLoad Time: 0.0022s | Model Forward Time: 0.5419s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0413 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.1560 | max_tensorboard_flush: 0.0006 | Memory: rss_MB: 2672.6 | uss_MB: 1528.6 | child_rss_MB: 22483.6 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:04:18,670] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:04:18,747] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 06:04:52,138] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 06:04:52,139] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:05:54,835] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 06:05:54,836] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:06:10,902] INFO : Step 17000/500000 rank0 | Train Loss: 9.3705 | Train Loss Unweighted: 4.0817 | Train Loss Components: Classification: 0.0335 | Classification_binary: 3.5859 | Regression_cos_phi: 0.0125 | Regression_energy: 0.2134 | Regression_eta: 0.0091 | Regression_pt: 0.2130 | Regression_sin_phi: 0.0143 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3753 | Classification_binary: 3.5859 | Regression_cos_phi: 0.1404 | Regression_energy: 2.2250 | Regression_eta: 0.0714 | Regression_pt: 2.8175 | Regression_sin_phi: 0.1551 | LR: 3.51e-05 | DataLoad Time: 0.0020s | Model Forward Time: 0.5792s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1853 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2668.9 | uss_MB: 1524.9 | child_rss_MB: 22483.8 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:06:58,296] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 06:06:58,296] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:08:01,575] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 06:08:01,575] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:08:03,854] INFO : Step 18000/500000 rank0 | Train Loss: 9.2232 | Train Loss Unweighted: 4.2970 | Train Loss Components: Classification: 0.0388 | Classification_binary: 3.8345 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1941 | Regression_eta: 0.0108 | Regression_pt: 0.1970 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4349 | Classification_binary: 3.8345 | Regression_cos_phi: 0.1055 | Regression_energy: 2.0237 | Regression_eta: 0.0852 | Regression_pt: 2.6063 | Regression_sin_phi: 0.1331 | LR: 3.69e-05 | DataLoad Time: 0.0003s | Model Forward Time: 0.5764s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0362 | max_loss: 0.0030 | max_backward: 0.0407 | max_optimizer_step: 0.0691 | max_tensorboard_logging: 0.1826 | max_tensorboard_flush: 0.0005 | Memory: rss_MB: 2670.1 | uss_MB: 1526.1 | child_rss_MB: 22484.4 | allocated_MB: 126.1 | reserved_MB: 650.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 06:08:03,856] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:08:03,934] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 06:09:07,424] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 06:09:07,424] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:58,912] INFO : Step 19000/500000 rank0 | Train Loss: 9.7826 | Train Loss Unweighted: 4.5104 | Train Loss Components: Classification: 0.0381 | Classification_binary: 4.0161 | Regression_cos_phi: 0.0101 | Regression_energy: 0.2145 | Regression_eta: 0.0093 | Regression_pt: 0.2121 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4266 | Classification_binary: 4.0161 | Regression_cos_phi: 0.1134 | Regression_energy: 2.2361 | Regression_eta: 0.0731 | Regression_pt: 2.8059 | Regression_sin_phi: 0.1114 | LR: 3.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5961s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0031 | max_backward: 0.0402 | max_optimizer_step: 0.0688 | max_tensorboard_logging: 0.2071 | max_tensorboard_flush: 0.0005 | Memory: rss_MB: 2672.7 | uss_MB: 1528.7 | child_rss_MB: 22484.0 | allocated_MB: 125.7 | reserved_MB: 650.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:10:10,974] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 06:10:10,974] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:13,764] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 06:11:13,764] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:53,053] INFO : Step 20000/500000 rank0 | Train Loss: 8.9256 | Train Loss Unweighted: 4.1463 | Train Loss Components: Classification: 0.0414 | Classification_binary: 3.6990 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1840 | Regression_eta: 0.0083 | Regression_pt: 0.1926 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4640 | Classification_binary: 3.6990 | Regression_cos_phi: 0.1162 | Regression_energy: 1.9181 | Regression_eta: 0.0657 | Regression_pt: 2.5486 | Regression_sin_phi: 0.1140 | LR: 4.07e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.6120s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0030 | max_backward: 0.0403 | max_optimizer_step: 0.0679 | max_tensorboard_logging: 0.2328 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2672.9 | uss_MB: 1528.8 | child_rss_MB: 22484.6 | allocated_MB: 125.7 | reserved_MB: 656.0 | inactive_split_MB: 24.3 +rank0 [2026-11-14 06:11:53,079] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:11:53,191] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:11:53,192] INFO : Running validation on rank0 +rank0 [2026-11-14 06:11:53,193] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:11:53,193] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:11:53,346] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 06:11:53,464] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:11:53,595] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:11:53,665] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:11:53,733] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:11:53,798] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:11:53,864] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:11:53,932] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:11:54,030] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:11:54,145] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:11:54,257] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:11:54,325] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:11:54,391] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:11:54,456] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:11:54,522] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:11:54,587] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:11:54,683] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:11:54,688] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:11:54,690] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:11:54,693] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:11:54,695] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:11:54,697] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:11:54,699] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:11:54,702] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:11:54,704] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:11:54,706] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:11:54,708] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:11:54,710] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:11:54,713] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:11:54,715] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:11:54,717] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:11:54,719] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:11:56,314] INFO : VALIDATION | Step=20000/500000 | Train Loss=8.9256 | Valid Loss=8.9078 | Train Loss Unweighted=4.1463 | Valid Loss Unweighted=4.2175 | Stale=0 | ETA=906.4m +rank0 [2026-11-14 06:11:56,314] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:11:56,974] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:11:57,003] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:11:57,057] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:11:57,560] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:11:57,816] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:11:58,067] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:11:58,320] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:11:58,572] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:11:58,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:11:59,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:11:59,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:11:59,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:11:59,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:12:00,094] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:12:00,349] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:12:00,602] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:12:00,856] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:12:01,107] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:12:01,330] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:12:01,392] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:12:01,402] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:12:01,991] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:12:02,389] INFO : Plotted number of elements +rank0 [2026-11-14 06:12:03,719] INFO : Plotted jets +rank0 [2026-11-14 06:12:04,701] INFO : Plotted jet ratio +rank0 [2026-11-14 06:12:30,938] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 06:12:30,938] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:33,271] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 06:13:33,271] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:56,444] INFO : Step 21000/500000 rank0 | Train Loss: 7.9236 | Train Loss Unweighted: 3.8248 | Train Loss Components: Classification: 0.0331 | Classification_binary: 3.4391 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1610 | Regression_eta: 0.0084 | Regression_pt: 0.1586 | Regression_sin_phi: 0.0135 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3705 | Classification_binary: 3.4391 | Regression_cos_phi: 0.1241 | Regression_energy: 1.6788 | Regression_eta: 0.0664 | Regression_pt: 2.0986 | Regression_sin_phi: 0.1462 | LR: 4.28e-05 | DataLoad Time: 0.0027s | Model Forward Time: 0.5509s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0028 | max_backward: 0.0438 | max_optimizer_step: 0.0689 | max_tensorboard_logging: 0.1501 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2699.6 | uss_MB: 1555.6 | child_rss_MB: 22486.9 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:14:35,667] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 06:14:35,667] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:37,834] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 06:15:37,834] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:47,321] INFO : Step 22000/500000 rank0 | Train Loss: 10.8922 | Train Loss Unweighted: 4.3936 | Train Loss Components: Classification: 0.0329 | Classification_binary: 3.7828 | Regression_cos_phi: 0.0098 | Regression_energy: 0.2782 | Regression_eta: 0.0147 | Regression_pt: 0.2645 | Regression_sin_phi: 0.0107 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3682 | Classification_binary: 3.7828 | Regression_cos_phi: 0.1100 | Regression_energy: 2.8998 | Regression_eta: 0.1162 | Regression_pt: 3.4995 | Regression_sin_phi: 0.1157 | LR: 4.50e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5989s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0404 | max_optimizer_step: 0.0692 | max_tensorboard_logging: 0.2073 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2700.3 | uss_MB: 1556.4 | child_rss_MB: 22487.4 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:15:47,323] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:15:47,403] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:16:40,836] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 06:16:40,836] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:39,589] INFO : Step 23000/500000 rank0 | Train Loss: 8.9377 | Train Loss Unweighted: 4.3616 | Train Loss Components: Classification: 0.0308 | Classification_binary: 3.9318 | Regression_cos_phi: 0.0132 | Regression_energy: 0.1818 | Regression_eta: 0.0103 | Regression_pt: 0.1829 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3450 | Classification_binary: 3.9318 | Regression_cos_phi: 0.1477 | Regression_energy: 1.8950 | Regression_eta: 0.0814 | Regression_pt: 2.4197 | Regression_sin_phi: 0.1172 | LR: 4.73e-05 | DataLoad Time: 0.0023s | Model Forward Time: 0.5848s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0410 | max_loss: 0.0029 | max_backward: 0.0411 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1898 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2703.4 | uss_MB: 1559.4 | child_rss_MB: 22486.1 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:17:44,250] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 06:17:44,250] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:46,997] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 06:18:46,997] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:19:31,680] INFO : Step 24000/500000 rank0 | Train Loss: 7.4872 | Train Loss Unweighted: 3.8002 | Train Loss Components: Classification: 0.0332 | Classification_binary: 3.4546 | Regression_cos_phi: 0.0049 | Regression_energy: 0.1482 | Regression_eta: 0.0055 | Regression_pt: 0.1467 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3723 | Classification_binary: 3.4546 | Regression_cos_phi: 0.0551 | Regression_energy: 1.5454 | Regression_eta: 0.0433 | Regression_pt: 1.9404 | Regression_sin_phi: 0.0761 | LR: 4.97e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.5598s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0026 | max_backward: 0.0396 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1727 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2703.5 | uss_MB: 1559.5 | child_rss_MB: 22487.6 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:19:31,682] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:19:31,764] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:19:50,317] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 06:19:50,317] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:52,988] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 06:20:52,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:21:24,019] INFO : Step 25000/500000 rank0 | Train Loss: 9.7542 | Train Loss Unweighted: 4.1255 | Train Loss Components: Classification: 0.0351 | Classification_binary: 3.5990 | Regression_cos_phi: 0.0170 | Regression_energy: 0.2231 | Regression_eta: 0.0097 | Regression_pt: 0.2301 | Regression_sin_phi: 0.0115 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3933 | Classification_binary: 3.5990 | Regression_cos_phi: 0.1900 | Regression_energy: 2.3260 | Regression_eta: 0.0765 | Regression_pt: 3.0448 | Regression_sin_phi: 0.1246 | LR: 5.22e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.6089s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0363 | max_loss: 0.0030 | max_backward: 0.0405 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.2093 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2699.6 | uss_MB: 1555.7 | child_rss_MB: 22486.9 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:21:56,685] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 06:21:56,685] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:59,499] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 06:22:59,499] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:23:16,387] INFO : Step 26000/500000 rank0 | Train Loss: 8.5531 | Train Loss Unweighted: 4.1961 | Train Loss Components: Classification: 0.0357 | Classification_binary: 3.7891 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1742 | Regression_eta: 0.0062 | Regression_pt: 0.1790 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3999 | Classification_binary: 3.7891 | Regression_cos_phi: 0.0585 | Regression_energy: 1.8160 | Regression_eta: 0.0486 | Regression_pt: 2.3683 | Regression_sin_phi: 0.0727 | LR: 5.47e-05 | DataLoad Time: 0.0016s | Model Forward Time: 0.6198s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0372 | max_loss: 0.0029 | max_backward: 0.0422 | max_optimizer_step: 0.0689 | max_tensorboard_logging: 0.2333 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2700.3 | uss_MB: 1556.3 | child_rss_MB: 22487.1 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:23:16,389] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:23:16,472] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:24:02,508] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 06:24:02,508] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:05,450] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 06:25:05,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:25:08,640] INFO : Step 27000/500000 rank0 | Train Loss: 7.7604 | Train Loss Unweighted: 3.8876 | Train Loss Components: Classification: 0.0353 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1500 | Regression_eta: 0.0079 | Regression_pt: 0.1558 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3957 | Classification_binary: 3.5247 | Regression_cos_phi: 0.0638 | Regression_energy: 1.5633 | Regression_eta: 0.0623 | Regression_pt: 2.0607 | Regression_sin_phi: 0.0899 | LR: 5.74e-05 | DataLoad Time: 0.0027s | Model Forward Time: 0.5751s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0365 | max_loss: 0.0029 | max_backward: 0.0407 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.1758 | max_tensorboard_flush: 0.0005 | Memory: rss_MB: 2703.4 | uss_MB: 1559.4 | child_rss_MB: 22485.5 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:26:09,410] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 06:26:09,410] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:27:01,561] INFO : Step 28000/500000 rank0 | Train Loss: 8.1155 | Train Loss Unweighted: 4.0403 | Train Loss Components: Classification: 0.0352 | Classification_binary: 3.6588 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1565 | Regression_eta: 0.0072 | Regression_pt: 0.1637 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3946 | Classification_binary: 3.6588 | Regression_cos_phi: 0.1163 | Regression_energy: 1.6315 | Regression_eta: 0.0565 | Regression_pt: 2.1656 | Regression_sin_phi: 0.0922 | LR: 6.01e-05 | DataLoad Time: 0.0020s | Model Forward Time: 0.5889s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0388 | max_loss: 0.0029 | max_backward: 0.0409 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1903 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2703.6 | uss_MB: 1559.7 | child_rss_MB: 22487.4 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:27:01,563] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:27:01,636] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:27:12,825] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 06:27:12,825] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:15,680] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 06:28:15,681] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:53,796] INFO : Step 29000/500000 rank0 | Train Loss: 8.1836 | Train Loss Unweighted: 3.8308 | Train Loss Components: Classification: 0.0368 | Classification_binary: 3.4204 | Regression_cos_phi: 0.0111 | Regression_energy: 0.1687 | Regression_eta: 0.0143 | Regression_pt: 0.1715 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4120 | Classification_binary: 3.4204 | Regression_cos_phi: 0.1241 | Regression_energy: 1.7590 | Regression_eta: 0.1125 | Regression_pt: 2.2695 | Regression_sin_phi: 0.0862 | LR: 6.29e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5854s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0030 | max_backward: 0.0453 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1819 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2703.6 | uss_MB: 1559.7 | child_rss_MB: 22486.2 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:29:18,900] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 06:29:18,900] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:21,790] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 06:30:21,790] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:30:47,311] INFO : Step 30000/500000 rank0 | Train Loss: 8.5306 | Train Loss Unweighted: 4.0331 | Train Loss Components: Classification: 0.0327 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1803 | Regression_eta: 0.0097 | Regression_pt: 0.1830 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3660 | Classification_binary: 3.6115 | Regression_cos_phi: 0.0920 | Regression_energy: 1.8792 | Regression_eta: 0.0762 | Regression_pt: 2.4215 | Regression_sin_phi: 0.0843 | LR: 6.58e-05 | DataLoad Time: 0.0017s | Model Forward Time: 0.6339s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0032 | max_backward: 0.0416 | max_optimizer_step: 0.0718 | max_tensorboard_logging: 0.2206 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2700.3 | uss_MB: 1556.3 | child_rss_MB: 22482.7 | allocated_MB: 125.7 | reserved_MB: 668.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:30:47,313] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:30:47,396] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:30:47,397] INFO : Running validation on rank0 +rank0 [2026-11-14 06:30:47,397] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:30:47,397] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:30:47,592] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 06:30:47,720] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:30:47,843] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:30:47,917] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:30:47,988] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:30:48,058] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:30:48,130] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:30:48,201] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:30:48,304] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:30:48,426] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:30:48,545] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:30:48,615] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:30:48,686] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:30:48,754] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:30:48,824] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:30:48,895] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:30:49,001] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:30:49,006] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:30:49,009] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:30:49,012] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:30:49,014] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:30:49,016] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:30:49,019] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:30:49,021] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:30:49,024] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:30:49,026] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:30:49,028] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:30:49,030] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:30:49,033] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:30:49,035] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:30:49,037] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:30:49,040] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:30:50,702] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.5306 | Valid Loss=8.6071 | Train Loss Unweighted=4.0331 | Valid Loss Unweighted=4.0722 | Stale=0 | ETA=887.9m +rank0 [2026-11-14 06:30:50,703] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:30:51,215] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:30:51,228] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:51,283] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:30:51,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:30:52,087] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:30:52,377] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:30:52,644] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:30:52,914] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:30:53,188] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:30:53,469] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:30:53,736] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:30:53,996] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:30:54,261] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:30:54,536] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:30:54,805] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:30:55,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:30:55,334] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:30:55,596] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:30:55,845] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:30:55,902] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:30:55,924] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:56,548] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:30:56,977] INFO : Plotted number of elements +rank0 [2026-11-14 06:30:58,417] INFO : Plotted jets +rank0 [2026-11-14 06:30:59,492] INFO : Plotted jet ratio +rank0 [2026-11-14 06:31:40,791] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 06:31:40,791] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:44,006] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 06:32:44,006] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:32:54,695] INFO : Step 31000/500000 rank0 | Train Loss: 8.3726 | Train Loss Unweighted: 3.9560 | Train Loss Components: Classification: 0.0371 | Classification_binary: 3.5384 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1659 | Regression_eta: 0.0193 | Regression_pt: 0.1743 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.4157 | Classification_binary: 3.5384 | Regression_cos_phi: 0.1149 | Regression_energy: 1.7291 | Regression_eta: 0.1521 | Regression_pt: 2.3057 | Regression_sin_phi: 0.1167 | LR: 6.88e-05 | DataLoad Time: 0.0012s | Model Forward Time: 0.6486s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.2221 | max_tensorboard_flush: 0.0006 | Memory: rss_MB: 2825.6 | uss_MB: 1681.7 | child_rss_MB: 22484.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:33:47,597] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 06:33:47,597] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:47,096] INFO : Step 32000/500000 rank0 | Train Loss: 9.0571 | Train Loss Unweighted: 4.1940 | Train Loss Components: Classification: 0.0326 | Classification_binary: 3.7385 | Regression_cos_phi: 0.0093 | Regression_energy: 0.1937 | Regression_eta: 0.0081 | Regression_pt: 0.1970 | Regression_sin_phi: 0.0148 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3648 | Classification_binary: 3.7385 | Regression_cos_phi: 0.1037 | Regression_energy: 2.0198 | Regression_eta: 0.0637 | Regression_pt: 2.6063 | Regression_sin_phi: 0.1603 | LR: 7.19e-05 | DataLoad Time: 0.0024s | Model Forward Time: 0.5816s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0032 | max_backward: 0.0414 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1771 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2825.6 | uss_MB: 1681.7 | child_rss_MB: 22485.4 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:34:47,098] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:34:47,174] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:34:50,988] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 06:34:50,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:54,456] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 06:35:54,456] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:39,870] INFO : Step 33000/500000 rank0 | Train Loss: 6.9426 | Train Loss Unweighted: 3.4761 | Train Loss Components: Classification: 0.0268 | Classification_binary: 3.1515 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1395 | Regression_eta: 0.0067 | Regression_pt: 0.1414 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3001 | Classification_binary: 3.1515 | Regression_cos_phi: 0.0587 | Regression_energy: 1.4546 | Regression_eta: 0.0527 | Regression_pt: 1.8712 | Regression_sin_phi: 0.0537 | LR: 7.51e-05 | DataLoad Time: 0.0013s | Model Forward Time: 0.5782s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0027 | max_backward: 0.0398 | max_optimizer_step: 0.0691 | max_tensorboard_logging: 0.1851 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2724.8 | uss_MB: 1580.9 | child_rss_MB: 22487.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:36:57,296] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 06:36:57,296] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:59,642] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 06:37:59,642] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:31,032] INFO : Step 34000/500000 rank0 | Train Loss: 7.4224 | Train Loss Unweighted: 3.5897 | Train Loss Components: Classification: 0.0325 | Classification_binary: 3.2303 | Regression_cos_phi: 0.0076 | Regression_energy: 0.1525 | Regression_eta: 0.0082 | Regression_pt: 0.1545 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3644 | Classification_binary: 3.2303 | Regression_cos_phi: 0.0850 | Regression_energy: 1.5900 | Regression_eta: 0.0646 | Regression_pt: 2.0441 | Regression_sin_phi: 0.0441 | LR: 7.83e-05 | DataLoad Time: 0.0015s | Model Forward Time: 0.5427s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0026 | max_backward: 0.0395 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1538 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2725.5 | uss_MB: 1581.6 | child_rss_MB: 22486.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:38:31,034] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:38:31,107] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:39:02,489] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 06:39:02,489] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:05,167] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 06:40:05,167] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:23,389] INFO : Step 35000/500000 rank0 | Train Loss: 7.5492 | Train Loss Unweighted: 3.7288 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.3725 | Regression_cos_phi: 0.0046 | Regression_energy: 0.1562 | Regression_eta: 0.0045 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0043 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3037 | Classification_binary: 3.3725 | Regression_cos_phi: 0.0520 | Regression_energy: 1.6286 | Regression_eta: 0.0357 | Regression_pt: 2.1099 | Regression_sin_phi: 0.0469 | LR: 8.16e-05 | DataLoad Time: 0.0018s | Model Forward Time: 0.5924s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0029 | max_backward: 0.0400 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.2036 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2729.6 | uss_MB: 1585.7 | child_rss_MB: 22485.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:41:08,318] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 06:41:08,319] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:11,703] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 06:42:11,703] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:15,712] INFO : Step 36000/500000 rank0 | Train Loss: 8.5782 | Train Loss Unweighted: 3.7415 | Train Loss Components: Classification: 0.0333 | Classification_binary: 3.2914 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1966 | Regression_eta: 0.0035 | Regression_pt: 0.2034 | Regression_sin_phi: 0.0065 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3729 | Classification_binary: 3.2914 | Regression_cos_phi: 0.0742 | Regression_energy: 2.0496 | Regression_eta: 0.0278 | Regression_pt: 2.6914 | Regression_sin_phi: 0.0708 | LR: 8.50e-05 | DataLoad Time: 0.0023s | Model Forward Time: 0.5265s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0377 | max_loss: 0.0030 | max_backward: 0.0399 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1334 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2729.7 | uss_MB: 1585.8 | child_rss_MB: 22488.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:42:15,714] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:42:15,790] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 06:43:15,194] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 06:43:15,194] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:44:08,035] INFO : Step 37000/500000 rank0 | Train Loss: 6.8225 | Train Loss Unweighted: 3.6000 | Train Loss Components: Classification: 0.0235 | Classification_binary: 3.2978 | Regression_cos_phi: 0.0072 | Regression_energy: 0.1269 | Regression_eta: 0.0059 | Regression_pt: 0.1289 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2634 | Classification_binary: 3.2978 | Regression_cos_phi: 0.0807 | Regression_energy: 1.3232 | Regression_eta: 0.0469 | Regression_pt: 1.7059 | Regression_sin_phi: 0.1046 | LR: 8.85e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5885s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0366 | max_loss: 0.0032 | max_backward: 0.0407 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.1894 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2729.7 | uss_MB: 1585.8 | child_rss_MB: 22488.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:44:18,369] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 06:44:18,369] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:45:21,713] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 06:45:21,714] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:46:01,065] INFO : Step 38000/500000 rank0 | Train Loss: 7.9930 | Train Loss Unweighted: 3.6223 | Train Loss Components: Classification: 0.0309 | Classification_binary: 3.2121 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1773 | Regression_eta: 0.0078 | Regression_pt: 0.1744 | Regression_sin_phi: 0.0113 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3461 | Classification_binary: 3.2121 | Regression_cos_phi: 0.0951 | Regression_energy: 1.8483 | Regression_eta: 0.0613 | Regression_pt: 2.3074 | Regression_sin_phi: 0.1227 | LR: 9.21e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5801s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0413 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1717 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2725.6 | uss_MB: 1581.7 | child_rss_MB: 22487.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:46:01,067] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:46:01,140] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 06:46:25,047] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 06:46:25,047] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:28,311] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 06:47:28,312] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:53,681] INFO : Step 39000/500000 rank0 | Train Loss: 9.5954 | Train Loss Unweighted: 4.3674 | Train Loss Components: Classification: 0.0336 | Classification_binary: 3.8771 | Regression_cos_phi: 0.0085 | Regression_energy: 0.2139 | Regression_eta: 0.0114 | Regression_pt: 0.2136 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3763 | Classification_binary: 3.8771 | Regression_cos_phi: 0.0952 | Regression_energy: 2.2303 | Regression_eta: 0.0896 | Regression_pt: 2.8263 | Regression_sin_phi: 0.1005 | LR: 9.57e-05 | DataLoad Time: 0.0010s | Model Forward Time: 0.6071s | Phase Times: max_pre_step_barrier: 0.0005 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0679 | max_tensorboard_logging: 0.2215 | max_tensorboard_flush: 0.0002 | Memory: rss_MB: 2729.6 | uss_MB: 1585.8 | child_rss_MB: 22489.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:48:31,459] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 06:48:31,459] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:34,413] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 06:49:34,413] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:45,961] INFO : Step 40000/500000 rank0 | Train Loss: 9.1555 | Train Loss Unweighted: 4.1032 | Train Loss Components: Classification: 0.0278 | Classification_binary: 3.6323 | Regression_cos_phi: 0.0045 | Regression_energy: 0.2108 | Regression_eta: 0.0060 | Regression_pt: 0.2142 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3113 | Classification_binary: 3.6323 | Regression_cos_phi: 0.0504 | Regression_energy: 2.1972 | Regression_eta: 0.0473 | Regression_pt: 2.8336 | Regression_sin_phi: 0.0834 | LR: 9.94e-05 | DataLoad Time: 0.0019s | Model Forward Time: 0.6079s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0398 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.2129 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2730.0 | uss_MB: 1586.1 | child_rss_MB: 22490.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:49:45,964] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 06:49:46,050] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 06:49:46,053] INFO : Running validation on rank0 +rank0 [2026-11-14 06:49:46,053] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:49:46,053] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:49:46,255] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 06:49:46,391] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:49:46,513] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:49:46,587] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:49:46,657] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:49:46,725] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:49:46,794] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:49:46,865] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:49:46,965] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:49:47,085] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:49:47,200] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:49:47,269] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:49:47,338] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:49:47,406] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:49:47,474] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:49:47,544] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:49:47,647] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:49:47,652] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:49:47,655] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:49:47,657] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:49:47,659] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:49:47,661] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:49:47,664] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:49:47,666] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:49:47,668] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:49:47,671] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:49:47,673] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:49:47,675] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:49:47,677] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:49:47,680] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:49:47,682] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:49:47,684] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:49:49,301] INFO : VALIDATION | Step=40000/500000 | Train Loss=9.1555 | Valid Loss=8.3071 | Train Loss Unweighted=4.1032 | Valid Loss Unweighted=3.8876 | Stale=0 | ETA=870.0m +rank0 [2026-11-14 06:49:49,302] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:49:49,769] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:49:49,784] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:49,838] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:49:50,361] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:49:50,625] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:49:50,888] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:49:51,147] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:49:51,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:49:51,665] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:49:51,923] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:49:52,181] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:49:52,435] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:49:52,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:49:52,973] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:49:53,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:49:53,491] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:49:53,751] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:49:54,010] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:49:54,241] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:49:54,300] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:49:54,309] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:54,897] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:49:55,309] INFO : Plotted number of elements +rank0 [2026-11-14 06:49:56,677] INFO : Plotted jets +rank0 [2026-11-14 06:49:57,681] INFO : Plotted jet ratio +rank0 [2026-11-14 06:50:49,978] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 06:50:49,978] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:51:50,347] INFO : Step 41000/500000 rank0 | Train Loss: 7.1656 | Train Loss Unweighted: 3.6709 | Train Loss Components: Classification: 0.0302 | Classification_binary: 3.3430 | Regression_cos_phi: 0.0073 | Regression_energy: 0.1376 | Regression_eta: 0.0060 | Regression_pt: 0.1380 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3384 | Classification_binary: 3.3430 | Regression_cos_phi: 0.0820 | Regression_energy: 1.4344 | Regression_eta: 0.0475 | Regression_pt: 1.8256 | Regression_sin_phi: 0.0947 | LR: 1.03e-04 | DataLoad Time: 0.0021s | Model Forward Time: 0.5875s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0027 | max_backward: 0.0400 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.1883 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2751.3 | uss_MB: 1607.5 | child_rss_MB: 22495.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:51:53,261] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 06:51:53,261] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:52:55,802] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 06:52:55,802] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:53:44,833] INFO : Step 42000/500000 rank0 | Train Loss: 6.9200 | Train Loss Unweighted: 3.4885 | Train Loss Components: Classification: 0.0284 | Classification_binary: 3.1661 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1318 | Regression_eta: 0.0072 | Regression_pt: 0.1346 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3178 | Classification_binary: 3.1661 | Regression_cos_phi: 0.1230 | Regression_energy: 1.3743 | Regression_eta: 0.0568 | Regression_pt: 1.7810 | Regression_sin_phi: 0.1009 | LR: 1.07e-04 | DataLoad Time: 0.0011s | Model Forward Time: 0.6297s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0379 | max_loss: 0.0032 | max_backward: 0.0415 | max_optimizer_step: 0.0725 | max_tensorboard_logging: 0.2135 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2752.0 | uss_MB: 1608.1 | child_rss_MB: 22493.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:53:44,835] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 06:53:44,919] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 06:54:02,120] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 06:54:02,120] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:55:05,618] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 06:55:05,618] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:55:38,203] INFO : Step 43000/500000 rank0 | Train Loss: 8.6636 | Train Loss Unweighted: 3.9669 | Train Loss Components: Classification: 0.0307 | Classification_binary: 3.5272 | Regression_cos_phi: 0.0116 | Regression_energy: 0.1916 | Regression_eta: 0.0069 | Regression_pt: 0.1899 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3438 | Classification_binary: 3.5272 | Regression_cos_phi: 0.1294 | Regression_energy: 1.9976 | Regression_eta: 0.0545 | Regression_pt: 2.5128 | Regression_sin_phi: 0.0982 | LR: 1.11e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.5455s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0372 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1509 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.1 | uss_MB: 1610.2 | child_rss_MB: 22492.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 28.3 +rank0 [2026-11-14 06:56:08,741] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 06:56:08,741] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:57:11,418] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 06:57:11,418] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:57:30,361] INFO : Step 44000/500000 rank0 | Train Loss: 6.6579 | Train Loss Unweighted: 3.4944 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.1944 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1198 | Regression_eta: 0.0159 | Regression_pt: 0.1241 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3096 | Classification_binary: 3.1944 | Regression_cos_phi: 0.0675 | Regression_energy: 1.2490 | Regression_eta: 0.1257 | Regression_pt: 1.6423 | Regression_sin_phi: 0.0695 | LR: 1.15e-04 | DataLoad Time: 0.0017s | Model Forward Time: 0.5799s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0028 | max_backward: 0.0413 | max_optimizer_step: 0.0705 | max_tensorboard_logging: 0.1701 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.8 | uss_MB: 1611.0 | child_rss_MB: 22494.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:57:30,363] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 06:57:30,441] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 06:58:14,890] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 06:58:14,890] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:59:17,641] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 06:59:17,641] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:59:22,975] INFO : Step 45000/500000 rank0 | Train Loss: 7.7431 | Train Loss Unweighted: 3.7460 | Train Loss Components: Classification: 0.0264 | Classification_binary: 3.3718 | Regression_cos_phi: 0.0058 | Regression_energy: 0.1599 | Regression_eta: 0.0093 | Regression_pt: 0.1659 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2961 | Classification_binary: 3.3718 | Regression_cos_phi: 0.0653 | Regression_energy: 1.6673 | Regression_eta: 0.0737 | Regression_pt: 2.1951 | Regression_sin_phi: 0.0738 | LR: 1.19e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.5812s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0401 | max_optimizer_step: 0.0696 | max_tensorboard_logging: 0.1811 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2754.9 | uss_MB: 1611.0 | child_rss_MB: 22493.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:00:21,185] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 07:00:21,185] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:01:14,788] INFO : Step 46000/500000 rank0 | Train Loss: 8.0185 | Train Loss Unweighted: 3.7094 | Train Loss Components: Classification: 0.0303 | Classification_binary: 3.3029 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1687 | Regression_eta: 0.0148 | Regression_pt: 0.1713 | Regression_sin_phi: 0.0127 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3391 | Classification_binary: 3.3029 | Regression_cos_phi: 0.0967 | Regression_energy: 1.7591 | Regression_eta: 0.1170 | Regression_pt: 2.2664 | Regression_sin_phi: 0.1374 | LR: 1.23e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.5745s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0030 | max_backward: 0.0413 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1847 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2752.1 | uss_MB: 1608.2 | child_rss_MB: 22495.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 56.3 +rank0 [2026-11-14 07:01:14,790] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:01:14,867] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 07:01:24,164] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 07:01:24,164] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:02:27,047] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 07:02:27,047] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:03:07,337] INFO : Step 47000/500000 rank0 | Train Loss: 6.3793 | Train Loss Unweighted: 3.3890 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.1070 | Regression_cos_phi: 0.0128 | Regression_energy: 0.1133 | Regression_eta: 0.0070 | Regression_pt: 0.1117 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3360 | Classification_binary: 3.1070 | Regression_cos_phi: 0.1432 | Regression_energy: 1.1811 | Regression_eta: 0.0556 | Regression_pt: 1.4784 | Regression_sin_phi: 0.0780 | LR: 1.27e-04 | DataLoad Time: 0.0015s | Model Forward Time: 0.5663s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.1774 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.2 | uss_MB: 1610.4 | child_rss_MB: 22494.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 18.3 +rank0 [2026-11-14 07:03:30,228] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 07:03:30,228] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:33,068] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 07:04:33,068] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:59,265] INFO : Step 48000/500000 rank0 | Train Loss: 6.5567 | Train Loss Unweighted: 3.3682 | Train Loss Components: Classification: 0.0277 | Classification_binary: 3.0666 | Regression_cos_phi: 0.0088 | Regression_energy: 0.1214 | Regression_eta: 0.0127 | Regression_pt: 0.1233 | Regression_sin_phi: 0.0078 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3107 | Classification_binary: 3.0666 | Regression_cos_phi: 0.0981 | Regression_energy: 1.2658 | Regression_eta: 0.1000 | Regression_pt: 1.6314 | Regression_sin_phi: 0.0842 | LR: 1.31e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.6064s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0401 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.2083 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.3 | uss_MB: 1610.5 | child_rss_MB: 22494.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:04:59,267] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:04:59,337] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 07:05:36,210] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 07:05:36,210] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:38,771] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 07:06:38,772] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:51,076] INFO : Step 49000/500000 rank0 | Train Loss: 7.3159 | Train Loss Unweighted: 3.5160 | Train Loss Components: Classification: 0.0300 | Classification_binary: 3.1603 | Regression_cos_phi: 0.0060 | Regression_energy: 0.1511 | Regression_eta: 0.0050 | Regression_pt: 0.1527 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3365 | Classification_binary: 3.1603 | Regression_cos_phi: 0.0677 | Regression_energy: 1.5751 | Regression_eta: 0.0394 | Regression_pt: 2.0196 | Regression_sin_phi: 0.1173 | LR: 1.36e-04 | DataLoad Time: 0.0021s | Model Forward Time: 0.5972s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0029 | max_backward: 0.0408 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1996 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2754.4 | uss_MB: 1610.5 | child_rss_MB: 22492.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:07:42,143] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 07:07:42,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:08:43,863] INFO : Step 50000/500000 rank0 | Train Loss: 6.4145 | Train Loss Unweighted: 3.4731 | Train Loss Components: Classification: 0.0276 | Classification_binary: 3.1958 | Regression_cos_phi: 0.0059 | Regression_energy: 0.1181 | Regression_eta: 0.0068 | Regression_pt: 0.1129 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3097 | Classification_binary: 3.1958 | Regression_cos_phi: 0.0666 | Regression_energy: 1.2309 | Regression_eta: 0.0537 | Regression_pt: 1.4943 | Regression_sin_phi: 0.0635 | LR: 1.40e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.6453s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0030 | max_backward: 0.0449 | max_optimizer_step: 0.0711 | max_tensorboard_logging: 0.2431 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2751.5 | uss_MB: 1607.7 | child_rss_MB: 22494.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:08:43,866] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:08:43,940] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 07:08:43,941] INFO : Running validation on rank0 +rank0 [2026-11-14 07:08:43,941] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:08:43,941] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:08:44,097] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 07:08:44,218] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:08:44,333] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:08:44,403] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:08:44,470] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:08:44,536] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:08:44,604] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:08:44,671] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:08:44,768] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:08:44,884] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:08:44,999] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:08:45,068] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:08:45,136] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:08:45,202] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:08:45,270] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:08:45,338] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:08:45,456] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:08:45,462] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:08:45,464] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:08:45,467] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:08:45,469] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:08:45,471] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:08:45,474] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:08:45,476] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:08:45,478] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:08:45,481] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:08:45,483] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:08:45,485] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:08:45,487] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:08:45,490] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:08:45,492] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:08:45,494] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:08:47,078] INFO : VALIDATION | Step=50000/500000 | Train Loss=6.4145 | Valid Loss=8.1470 | Train Loss Unweighted=3.4731 | Valid Loss Unweighted=3.7909 | Stale=0 | ETA=851.5m +rank0 [2026-11-14 07:08:47,079] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:08:47,701] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:08:47,726] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:47,781] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:08:48,322] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:08:48,587] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:08:48,843] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:08:49,101] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:08:49,360] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:08:49,613] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:08:49,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:08:50,152] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:08:50,406] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:08:50,669] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:08:50,920] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:08:51,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:08:51,432] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:08:51,684] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:08:51,949] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:08:52,182] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:08:52,247] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:08:52,259] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:52,860] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:08:53,262] INFO : Plotted number of elements +rank0 [2026-11-14 07:08:54,617] INFO : Plotted jets +rank0 [2026-11-14 07:08:55,625] INFO : Plotted jet ratio +rank0 [2026-11-14 07:08:58,131] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 07:08:58,131] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:10:01,338] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 07:10:01,338] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:10:49,244] INFO : Step 51000/500000 rank0 | Train Loss: 7.8427 | Train Loss Unweighted: 3.7592 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.3727 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1565 | Regression_eta: 0.0195 | Regression_pt: 0.1627 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3121 | Classification_binary: 3.3727 | Regression_cos_phi: 0.1052 | Regression_energy: 1.6316 | Regression_eta: 0.1539 | Regression_pt: 2.1531 | Regression_sin_phi: 0.1141 | LR: 1.44e-04 | DataLoad Time: 0.0009s | Model Forward Time: 0.6096s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0028 | max_backward: 0.0404 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.2070 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2884.0 | uss_MB: 1740.2 | child_rss_MB: 22496.6 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:11:05,025] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 07:11:05,025] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:12:07,758] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 07:12:07,758] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:12:41,381] INFO : Step 52000/500000 rank0 | Train Loss: 6.9050 | Train Loss Unweighted: 3.4499 | Train Loss Components: Classification: 0.0351 | Classification_binary: 3.1232 | Regression_cos_phi: 0.0110 | Regression_energy: 0.1309 | Regression_eta: 0.0106 | Regression_pt: 0.1295 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3928 | Classification_binary: 3.1232 | Regression_cos_phi: 0.1231 | Regression_energy: 1.3642 | Regression_eta: 0.0838 | Regression_pt: 1.7135 | Regression_sin_phi: 0.1044 | LR: 1.49e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5517s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0406 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.1488 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2884.0 | uss_MB: 1740.2 | child_rss_MB: 22498.2 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:12:41,383] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:12:41,457] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:13:11,041] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 07:13:11,041] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:14:13,717] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 07:14:13,717] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:14:33,286] INFO : Step 53000/500000 rank0 | Train Loss: 6.4875 | Train Loss Unweighted: 3.2492 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.9457 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1271 | Regression_eta: 0.0085 | Regression_pt: 0.1337 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2515 | Classification_binary: 2.9457 | Regression_cos_phi: 0.0680 | Regression_energy: 1.3253 | Regression_eta: 0.0668 | Regression_pt: 1.7692 | Regression_sin_phi: 0.0610 | LR: 1.53e-04 | DataLoad Time: 0.0025s | Model Forward Time: 0.5946s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0416 | max_optimizer_step: 0.0705 | max_tensorboard_logging: 0.1885 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2884.1 | uss_MB: 1740.2 | child_rss_MB: 22499.0 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:15:17,490] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 07:15:17,491] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:22,983] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 07:16:22,984] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:28,967] INFO : Step 54000/500000 rank0 | Train Loss: 6.7482 | Train Loss Unweighted: 3.4032 | Train Loss Components: Classification: 0.0321 | Classification_binary: 3.0859 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1302 | Regression_eta: 0.0130 | Regression_pt: 0.1259 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3600 | Classification_binary: 3.0859 | Regression_cos_phi: 0.0786 | Regression_energy: 1.3570 | Regression_eta: 0.1024 | Regression_pt: 1.6657 | Regression_sin_phi: 0.0986 | LR: 1.58e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.6236s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0373 | max_loss: 0.0030 | max_backward: 0.0416 | max_optimizer_step: 0.0713 | max_tensorboard_logging: 0.2213 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2777.6 | uss_MB: 1633.8 | child_rss_MB: 22497.5 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:16:28,969] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:16:29,040] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:17:26,859] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 07:17:26,859] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:18:22,050] INFO : Step 55000/500000 rank0 | Train Loss: 6.3495 | Train Loss Unweighted: 3.2879 | Train Loss Components: Classification: 0.0279 | Classification_binary: 3.0004 | Regression_cos_phi: 0.0071 | Regression_energy: 0.1185 | Regression_eta: 0.0064 | Regression_pt: 0.1204 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3123 | Classification_binary: 3.0004 | Regression_cos_phi: 0.0799 | Regression_energy: 1.2350 | Regression_eta: 0.0507 | Regression_pt: 1.5934 | Regression_sin_phi: 0.0778 | LR: 1.62e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.5676s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0030 | max_backward: 0.0406 | max_optimizer_step: 0.0683 | max_tensorboard_logging: 0.1756 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2781.5 | uss_MB: 1637.7 | child_rss_MB: 22496.7 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:18:30,547] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 07:18:30,547] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:19:33,636] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 07:19:33,636] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:20:14,689] INFO : Step 56000/500000 rank0 | Train Loss: 6.3309 | Train Loss Unweighted: 3.1427 | Train Loss Components: Classification: 0.0263 | Classification_binary: 2.8434 | Regression_cos_phi: 0.0065 | Regression_energy: 0.1283 | Regression_eta: 0.0064 | Regression_pt: 0.1271 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2945 | Classification_binary: 2.8434 | Regression_cos_phi: 0.0729 | Regression_energy: 1.3378 | Regression_eta: 0.0507 | Regression_pt: 1.6822 | Regression_sin_phi: 0.0494 | LR: 1.67e-04 | DataLoad Time: 0.0017s | Model Forward Time: 0.5696s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0029 | max_backward: 0.0420 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1786 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2782.0 | uss_MB: 1638.1 | child_rss_MB: 22497.8 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:20:14,692] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:20:14,769] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:20:37,026] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 07:20:37,026] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:21:39,829] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 07:21:39,830] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:22:07,122] INFO : Step 57000/500000 rank0 | Train Loss: 6.1664 | Train Loss Unweighted: 3.1832 | Train Loss Components: Classification: 0.0249 | Classification_binary: 2.8996 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1159 | Regression_eta: 0.0132 | Regression_pt: 0.1120 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2785 | Classification_binary: 2.8996 | Regression_cos_phi: 0.1001 | Regression_energy: 1.2082 | Regression_eta: 0.1037 | Regression_pt: 1.4821 | Regression_sin_phi: 0.0942 | LR: 1.72e-04 | DataLoad Time: 0.0021s | Model Forward Time: 0.5878s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0030 | max_backward: 0.0417 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1951 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2782.1 | uss_MB: 1638.2 | child_rss_MB: 22491.3 | allocated_MB: 125.7 | reserved_MB: 628.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:22:43,591] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 07:22:43,591] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:23:47,213] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 07:23:47,214] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:24:00,379] INFO : Step 58000/500000 rank0 | Train Loss: 6.3335 | Train Loss Unweighted: 3.4289 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.1532 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1110 | Regression_eta: 0.0110 | Regression_pt: 0.1078 | Regression_sin_phi: 0.0108 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3035 | Classification_binary: 3.1532 | Regression_cos_phi: 0.0899 | Regression_energy: 1.1569 | Regression_eta: 0.0867 | Regression_pt: 1.4266 | Regression_sin_phi: 0.1167 | LR: 1.76e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.5348s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0027 | max_backward: 0.0407 | max_optimizer_step: 0.0694 | max_tensorboard_logging: 0.1379 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2777.1 | uss_MB: 1633.3 | child_rss_MB: 22491.2 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:24:00,381] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:24:00,460] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:24:50,830] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 07:24:50,830] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:25:53,150] INFO : Step 59000/500000 rank0 | Train Loss: 5.7660 | Train Loss Unweighted: 3.1763 | Train Loss Components: Classification: 0.0251 | Classification_binary: 2.9333 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0975 | Regression_eta: 0.0064 | Regression_pt: 0.1034 | Regression_sin_phi: 0.0046 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2807 | Classification_binary: 2.9333 | Regression_cos_phi: 0.0672 | Regression_energy: 1.0168 | Regression_eta: 0.0505 | Regression_pt: 1.3674 | Regression_sin_phi: 0.0501 | LR: 1.81e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.5993s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0413 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.2004 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2781.5 | uss_MB: 1637.7 | child_rss_MB: 22491.1 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:25:54,275] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 07:25:54,275] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:26:56,506] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 07:26:56,506] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:27:45,342] INFO : Step 60000/500000 rank0 | Train Loss: 6.4657 | Train Loss Unweighted: 3.3919 | Train Loss Components: Classification: 0.0221 | Classification_binary: 3.1040 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1219 | Regression_eta: 0.0058 | Regression_pt: 0.1242 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2478 | Classification_binary: 3.1040 | Regression_cos_phi: 0.0930 | Regression_energy: 1.2707 | Regression_eta: 0.0454 | Regression_pt: 1.6436 | Regression_sin_phi: 0.0612 | LR: 1.86e-04 | DataLoad Time: 0.0012s | Model Forward Time: 0.6041s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0030 | max_backward: 0.0408 | max_optimizer_step: 0.0711 | max_tensorboard_logging: 0.2022 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2781.7 | uss_MB: 1637.9 | child_rss_MB: 22490.7 | allocated_MB: 125.7 | reserved_MB: 640.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:27:45,344] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:27:45,426] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:27:45,428] INFO : Running validation on rank0 +rank0 [2026-11-14 07:27:45,428] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:27:45,428] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:27:45,591] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 07:27:45,712] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:27:45,828] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:27:45,897] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:27:45,965] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:27:46,034] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:27:46,104] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:27:46,173] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:27:46,275] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:27:46,392] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:27:46,506] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:27:46,574] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:27:46,643] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:27:46,710] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:27:46,778] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:27:46,847] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:27:46,958] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:27:46,964] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:27:46,966] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:27:46,969] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:27:46,972] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:27:46,974] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:27:46,976] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:27:46,978] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:27:46,980] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:27:46,983] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:27:46,985] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:27:46,987] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:27:46,990] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:27:46,992] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:27:46,994] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:27:46,997] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:27:48,526] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.4657 | Valid Loss=8.4191 | Train Loss Unweighted=3.3919 | Valid Loss Unweighted=3.8145 | Stale=1 | ETA=833.3m +rank0 [2026-11-14 07:27:48,527] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:27:49,275] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:27:49,288] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:49,345] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:27:49,885] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:27:50,145] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:27:50,402] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:27:50,660] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:27:50,921] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:27:51,177] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:27:51,438] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:27:51,695] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:27:51,965] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:27:52,239] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:27:52,495] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:27:52,756] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:27:53,009] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:27:53,264] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:27:53,517] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:27:53,747] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:27:53,813] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:27:53,832] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:54,433] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:27:54,839] INFO : Plotted number of elements +rank0 [2026-11-14 07:27:56,220] INFO : Plotted jets +rank0 [2026-11-14 07:27:57,227] INFO : Plotted jet ratio +rank0 [2026-11-14 07:28:12,493] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 07:28:12,493] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:14,937] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 07:29:14,937] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:49,667] INFO : Step 61000/500000 rank0 | Train Loss: 5.7818 | Train Loss Unweighted: 3.0853 | Train Loss Components: Classification: 0.0214 | Classification_binary: 2.8315 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1037 | Regression_eta: 0.0063 | Regression_pt: 0.1047 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2399 | Classification_binary: 2.8315 | Regression_cos_phi: 0.0738 | Regression_energy: 1.0807 | Regression_eta: 0.0497 | Regression_pt: 1.3858 | Regression_sin_phi: 0.1203 | LR: 1.91e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.5516s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0029 | max_backward: 0.0410 | max_optimizer_step: 0.0715 | max_tensorboard_logging: 0.1395 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2911.4 | uss_MB: 1767.6 | child_rss_MB: 22491.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 26.3 +rank0 [2026-11-14 07:30:18,200] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 07:30:18,200] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:31:20,430] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 07:31:20,430] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:31:41,362] INFO : Step 62000/500000 rank0 | Train Loss: 7.1917 | Train Loss Unweighted: 3.7193 | Train Loss Components: Classification: 0.0252 | Classification_binary: 3.3948 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1383 | Regression_eta: 0.0063 | Regression_pt: 0.1439 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2822 | Classification_binary: 3.3948 | Regression_cos_phi: 0.0760 | Regression_energy: 1.4416 | Regression_eta: 0.0494 | Regression_pt: 1.9033 | Regression_sin_phi: 0.0445 | LR: 1.95e-04 | DataLoad Time: 0.0020s | Model Forward Time: 0.6371s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0396 | max_loss: 0.0031 | max_backward: 0.0409 | max_optimizer_step: 0.0719 | max_tensorboard_logging: 0.2221 | max_tensorboard_flush: 0.0004 | Memory: rss_MB: 2803.5 | uss_MB: 1659.7 | child_rss_MB: 22489.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:31:41,365] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:31:41,440] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:32:24,668] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 07:32:24,668] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:33:27,858] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 07:33:27,858] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:33:34,690] INFO : Step 63000/500000 rank0 | Train Loss: 5.8449 | Train Loss Unweighted: 3.3032 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.0623 | Regression_cos_phi: 0.0092 | Regression_energy: 0.0938 | Regression_eta: 0.0088 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0088 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3030 | Classification_binary: 3.0623 | Regression_cos_phi: 0.1031 | Regression_energy: 0.9782 | Regression_eta: 0.0693 | Regression_pt: 1.2332 | Regression_sin_phi: 0.0958 | LR: 2.00e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.6138s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0369 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.2008 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2806.9 | uss_MB: 1663.1 | child_rss_MB: 22490.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:34:31,349] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 07:34:31,349] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:35:27,229] INFO : Step 64000/500000 rank0 | Train Loss: 5.5282 | Train Loss Unweighted: 3.0770 | Train Loss Components: Classification: 0.0248 | Classification_binary: 2.8426 | Regression_cos_phi: 0.0049 | Regression_energy: 0.0936 | Regression_eta: 0.0153 | Regression_pt: 0.0912 | Regression_sin_phi: 0.0048 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2775 | Classification_binary: 2.8426 | Regression_cos_phi: 0.0545 | Regression_energy: 0.9757 | Regression_eta: 0.1202 | Regression_pt: 1.2061 | Regression_sin_phi: 0.0516 | LR: 2.05e-04 | DataLoad Time: 0.0022s | Model Forward Time: 0.5679s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0374 | max_loss: 0.0030 | max_backward: 0.0419 | max_optimizer_step: 0.0705 | max_tensorboard_logging: 0.1673 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.9 | uss_MB: 1663.1 | child_rss_MB: 22489.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:35:27,232] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:35:27,314] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:35:34,924] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 07:35:34,924] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:36:38,004] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 07:36:38,004] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:37:20,826] INFO : Step 65000/500000 rank0 | Train Loss: 6.2154 | Train Loss Unweighted: 3.2558 | Train Loss Components: Classification: 0.0284 | Classification_binary: 2.9784 | Regression_cos_phi: 0.0047 | Regression_energy: 0.1181 | Regression_eta: 0.0040 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0050 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3185 | Classification_binary: 2.9784 | Regression_cos_phi: 0.0532 | Regression_energy: 1.2314 | Regression_eta: 0.0316 | Regression_pt: 1.5485 | Regression_sin_phi: 0.0539 | LR: 2.10e-04 | DataLoad Time: 0.0022s | Model Forward Time: 0.5948s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0030 | max_backward: 0.0448 | max_optimizer_step: 0.0730 | max_tensorboard_logging: 0.1813 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2807.0 | uss_MB: 1663.2 | child_rss_MB: 22491.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:37:43,375] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 07:37:43,375] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:38:48,408] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 07:38:48,408] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:39:16,077] INFO : Step 66000/500000 rank0 | Train Loss: 6.3929 | Train Loss Unweighted: 3.1117 | Train Loss Components: Classification: 0.0350 | Classification_binary: 2.8030 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1250 | Regression_eta: 0.0079 | Regression_pt: 0.1268 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3925 | Classification_binary: 2.8030 | Regression_cos_phi: 0.0713 | Regression_energy: 1.3026 | Regression_eta: 0.0625 | Regression_pt: 1.6778 | Regression_sin_phi: 0.0832 | LR: 2.15e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.5940s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0362 | max_loss: 0.0029 | max_backward: 0.0402 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1992 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.0 | uss_MB: 1659.2 | child_rss_MB: 22493.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:39:16,079] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:39:16,155] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:39:51,590] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 07:39:51,590] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:40:54,351] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 07:40:54,351] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:41:08,495] INFO : Step 67000/500000 rank0 | Train Loss: 5.2425 | Train Loss Unweighted: 2.8737 | Train Loss Components: Classification: 0.0246 | Classification_binary: 2.6501 | Regression_cos_phi: 0.0071 | Regression_energy: 0.0875 | Regression_eta: 0.0063 | Regression_pt: 0.0888 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2758 | Classification_binary: 2.6501 | Regression_cos_phi: 0.0797 | Regression_energy: 0.9123 | Regression_eta: 0.0494 | Regression_pt: 1.1745 | Regression_sin_phi: 0.1007 | LR: 2.20e-04 | DataLoad Time: 0.0013s | Model Forward Time: 0.5675s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0412 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1712 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.3 | uss_MB: 1662.5 | child_rss_MB: 22493.6 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:41:57,982] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 07:41:57,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:43:01,840] INFO : Step 68000/500000 rank0 | Train Loss: 5.8962 | Train Loss Unweighted: 3.2217 | Train Loss Components: Classification: 0.0247 | Classification_binary: 2.9707 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1025 | Regression_eta: 0.0051 | Regression_pt: 0.1053 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2767 | Classification_binary: 2.9707 | Regression_cos_phi: 0.0747 | Regression_energy: 1.0688 | Regression_eta: 0.0402 | Regression_pt: 1.3935 | Regression_sin_phi: 0.0716 | LR: 2.25e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.5722s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0367 | max_loss: 0.0030 | max_backward: 0.0409 | max_optimizer_step: 0.0735 | max_tensorboard_logging: 0.1651 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.4 | uss_MB: 1662.6 | child_rss_MB: 22491.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:43:01,842] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:43:01,922] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 07:43:02,143] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 07:43:02,143] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:44:05,189] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 07:44:05,189] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:44:54,528] INFO : Step 69000/500000 rank0 | Train Loss: 5.4393 | Train Loss Unweighted: 2.8107 | Train Loss Components: Classification: 0.0274 | Classification_binary: 2.5638 | Regression_cos_phi: 0.0052 | Regression_energy: 0.1041 | Regression_eta: 0.0039 | Regression_pt: 0.1012 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.3072 | Classification_binary: 2.5638 | Regression_cos_phi: 0.0588 | Regression_energy: 1.0852 | Regression_eta: 0.0308 | Regression_pt: 1.3383 | Regression_sin_phi: 0.0553 | LR: 2.30e-04 | DataLoad Time: 0.0018s | Model Forward Time: 0.5819s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0029 | max_backward: 0.0407 | max_optimizer_step: 0.0701 | max_tensorboard_logging: 0.1865 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2806.5 | uss_MB: 1662.7 | child_rss_MB: 22492.5 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:45:08,583] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 07:45:08,583] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:46:11,806] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 07:46:11,806] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:46:47,602] INFO : Step 70000/500000 rank0 | Train Loss: 4.8632 | Train Loss Unweighted: 2.7565 | Train Loss Components: Classification: 0.0225 | Classification_binary: 2.5584 | Regression_cos_phi: 0.0042 | Regression_energy: 0.0802 | Regression_eta: 0.0041 | Regression_pt: 0.0810 | Regression_sin_phi: 0.0060 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2517 | Classification_binary: 2.5584 | Regression_cos_phi: 0.0467 | Regression_energy: 0.8364 | Regression_eta: 0.0327 | Regression_pt: 1.0719 | Regression_sin_phi: 0.0654 | LR: 2.35e-04 | DataLoad Time: 0.0022s | Model Forward Time: 0.6094s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0030 | max_backward: 0.0415 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.2087 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2803.0 | uss_MB: 1659.2 | child_rss_MB: 22493.8 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:46:47,605] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:46:47,684] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 07:46:47,685] INFO : Running validation on rank0 +rank0 [2026-11-14 07:46:47,685] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:46:47,685] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:46:47,843] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 07:46:47,964] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:46:48,082] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:46:48,153] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:46:48,221] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:46:48,291] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:46:48,362] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:46:48,433] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:46:48,538] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:46:48,655] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:46:48,771] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:46:48,839] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:46:48,906] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:46:48,974] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:46:49,045] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:46:49,115] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:46:49,225] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:46:49,231] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:46:49,233] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:46:49,236] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:46:49,238] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:46:49,241] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:46:49,243] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:46:49,246] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:46:49,248] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:46:49,250] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:46:49,253] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:46:49,255] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:46:49,258] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:46:49,260] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:46:49,262] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:46:49,265] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:46:50,799] INFO : VALIDATION | Step=70000/500000 | Train Loss=4.8632 | Valid Loss=8.7554 | Train Loss Unweighted=2.7565 | Valid Loss Unweighted=3.9366 | Stale=2 | ETA=815.0m +rank0 [2026-11-14 07:46:50,800] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:46:51,313] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:46:51,326] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:46:51,382] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:46:51,929] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:46:52,189] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:46:52,448] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:46:52,706] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:46:52,959] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:46:53,212] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:46:53,468] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:46:53,733] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:46:53,998] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:46:54,258] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:46:54,516] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:46:54,776] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:46:55,032] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:46:55,287] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:46:55,540] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:46:55,770] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:46:55,830] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:46:55,852] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:46:56,453] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:46:56,848] INFO : Plotted number of elements +rank0 [2026-11-14 07:46:58,214] INFO : Plotted jets +rank0 [2026-11-14 07:46:59,218] INFO : Plotted jet ratio +rank0 [2026-11-14 07:47:27,722] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 07:47:27,722] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:48:31,245] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 07:48:31,245] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:48:52,796] INFO : Step 71000/500000 rank0 | Train Loss: 5.1706 | Train Loss Unweighted: 2.8523 | Train Loss Components: Classification: 0.0215 | Classification_binary: 2.6305 | Regression_cos_phi: 0.0080 | Regression_energy: 0.0835 | Regression_eta: 0.0139 | Regression_pt: 0.0844 | Regression_sin_phi: 0.0105 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2409 | Classification_binary: 2.6305 | Regression_cos_phi: 0.0891 | Regression_energy: 0.8709 | Regression_eta: 0.1093 | Regression_pt: 1.1162 | Regression_sin_phi: 0.1136 | LR: 2.40e-04 | DataLoad Time: 0.0014s | Model Forward Time: 0.5760s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0399 | max_optimizer_step: 0.0704 | max_tensorboard_logging: 0.1725 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2936.9 | uss_MB: 1793.1 | child_rss_MB: 22495.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:49:35,048] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 07:49:35,048] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:50:38,082] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 07:50:38,082] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:50:45,814] INFO : Step 72000/500000 rank0 | Train Loss: 6.3488 | Train Loss Unweighted: 3.4243 | Train Loss Components: Classification: 0.0264 | Classification_binary: 3.1491 | Regression_cos_phi: 0.0092 | Regression_energy: 0.1127 | Regression_eta: 0.0064 | Regression_pt: 0.1125 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2958 | Classification_binary: 3.1491 | Regression_cos_phi: 0.1035 | Regression_energy: 1.1753 | Regression_eta: 0.0503 | Regression_pt: 1.4885 | Regression_sin_phi: 0.0864 | LR: 2.45e-04 | DataLoad Time: 0.0022s | Model Forward Time: 0.6069s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0380 | max_loss: 0.0030 | max_backward: 0.0426 | max_optimizer_step: 0.0718 | max_tensorboard_logging: 0.2036 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2936.9 | uss_MB: 1793.1 | child_rss_MB: 22496.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:50:45,816] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 07:50:45,892] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 07:51:41,509] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 07:51:41,509] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:52:38,493] INFO : Step 73000/500000 rank0 | Train Loss: 4.7876 | Train Loss Unweighted: 2.8118 | Train Loss Components: Classification: 0.0212 | Classification_binary: 2.6239 | Regression_cos_phi: 0.0108 | Regression_energy: 0.0665 | Regression_eta: 0.0076 | Regression_pt: 0.0690 | Regression_sin_phi: 0.0129 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2373 | Classification_binary: 2.6239 | Regression_cos_phi: 0.1208 | Regression_energy: 0.6932 | Regression_eta: 0.0602 | Regression_pt: 0.9128 | Regression_sin_phi: 0.1394 | LR: 2.50e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.5690s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0365 | max_loss: 0.0037 | max_backward: 0.0409 | max_optimizer_step: 0.0706 | max_tensorboard_logging: 0.1599 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2936.9 | uss_MB: 1793.2 | child_rss_MB: 22497.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:52:45,241] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 07:52:45,242] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:53:46,894] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 07:53:46,894] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:54:29,414] INFO : Step 74000/500000 rank0 | Train Loss: 4.6294 | Train Loss Unweighted: 2.6329 | Train Loss Components: Classification: 0.0203 | Classification_binary: 2.4446 | Regression_cos_phi: 0.0041 | Regression_energy: 0.0772 | Regression_eta: 0.0050 | Regression_pt: 0.0760 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2278 | Classification_binary: 2.4446 | Regression_cos_phi: 0.0457 | Regression_energy: 0.8045 | Regression_eta: 0.0395 | Regression_pt: 1.0055 | Regression_sin_phi: 0.0618 | LR: 2.55e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.6062s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0405 | max_optimizer_step: 0.0715 | max_tensorboard_logging: 0.1984 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2827.1 | uss_MB: 1683.3 | child_rss_MB: 22495.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:54:29,416] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 07:54:29,494] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 07:54:50,484] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 07:54:50,485] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:55:53,177] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 07:55:53,177] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:56:21,917] INFO : Step 75000/500000 rank0 | Train Loss: 5.2123 | Train Loss Unweighted: 2.9845 | Train Loss Components: Classification: 0.0236 | Classification_binary: 2.7712 | Regression_cos_phi: 0.0082 | Regression_energy: 0.0832 | Regression_eta: 0.0113 | Regression_pt: 0.0774 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2645 | Classification_binary: 2.7712 | Regression_cos_phi: 0.0923 | Regression_energy: 0.8676 | Regression_eta: 0.0891 | Regression_pt: 1.0237 | Regression_sin_phi: 0.1039 | LR: 2.60e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5381s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0030 | max_backward: 0.0404 | max_optimizer_step: 0.0680 | max_tensorboard_logging: 0.1549 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2829.8 | uss_MB: 1686.0 | child_rss_MB: 22496.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:56:56,154] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 07:56:56,154] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:57:58,864] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 07:57:58,864] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:58:13,849] INFO : Step 76000/500000 rank0 | Train Loss: 5.0237 | Train Loss Unweighted: 2.8620 | Train Loss Components: Classification: 0.0185 | Classification_binary: 2.6563 | Regression_cos_phi: 0.0074 | Regression_energy: 0.0807 | Regression_eta: 0.0114 | Regression_pt: 0.0818 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2067 | Classification_binary: 2.6563 | Regression_cos_phi: 0.0827 | Regression_energy: 0.8415 | Regression_eta: 0.0896 | Regression_pt: 1.0826 | Regression_sin_phi: 0.0643 | LR: 2.65e-04 | DataLoad Time: 0.0019s | Model Forward Time: 0.6040s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0693 | max_tensorboard_logging: 0.2093 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2829.8 | uss_MB: 1686.1 | child_rss_MB: 22494.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:58:13,851] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 07:58:13,922] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 07:59:02,063] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 07:59:02,063] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:05,774] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 08:00:05,775] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:06,991] INFO : Step 77000/500000 rank0 | Train Loss: 4.7542 | Train Loss Unweighted: 2.7273 | Train Loss Components: Classification: 0.0205 | Classification_binary: 2.5323 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0758 | Regression_eta: 0.0134 | Regression_pt: 0.0712 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2298 | Classification_binary: 2.5323 | Regression_cos_phi: 0.0651 | Regression_energy: 0.7905 | Regression_eta: 0.1055 | Regression_pt: 0.9415 | Regression_sin_phi: 0.0896 | LR: 2.70e-04 | DataLoad Time: 0.0003s | Model Forward Time: 0.6037s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0379 | max_loss: 0.0032 | max_backward: 0.0430 | max_optimizer_step: 0.0734 | max_tensorboard_logging: 0.1861 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.0 | uss_MB: 1686.3 | child_rss_MB: 22495.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:01:11,545] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 08:01:11,545] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:01,287] INFO : Step 78000/500000 rank0 | Train Loss: 4.5251 | Train Loss Unweighted: 2.7048 | Train Loss Components: Classification: 0.0181 | Classification_binary: 2.5321 | Regression_cos_phi: 0.0090 | Regression_energy: 0.0700 | Regression_eta: 0.0051 | Regression_pt: 0.0650 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2022 | Classification_binary: 2.5321 | Regression_cos_phi: 0.1004 | Regression_energy: 0.7297 | Regression_eta: 0.0402 | Regression_pt: 0.8598 | Regression_sin_phi: 0.0608 | LR: 2.75e-04 | DataLoad Time: 0.0010s | Model Forward Time: 0.6044s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0403 | max_optimizer_step: 0.0707 | max_tensorboard_logging: 0.1896 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.0 | uss_MB: 1686.2 | child_rss_MB: 22496.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:02:01,289] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 08:02:01,367] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 08:02:14,628] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 08:02:14,628] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:03:17,382] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 08:03:17,383] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:03:53,226] INFO : Step 79000/500000 rank0 | Train Loss: 4.3472 | Train Loss Unweighted: 2.4582 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.2808 | Regression_cos_phi: 0.0050 | Regression_energy: 0.0721 | Regression_eta: 0.0044 | Regression_pt: 0.0739 | Regression_sin_phi: 0.0034 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2105 | Classification_binary: 2.2808 | Regression_cos_phi: 0.0557 | Regression_energy: 0.7512 | Regression_eta: 0.0344 | Regression_pt: 0.9783 | Regression_sin_phi: 0.0363 | LR: 2.80e-04 | DataLoad Time: 0.0015s | Model Forward Time: 0.6502s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0697 | max_tensorboard_logging: 0.2391 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2829.7 | uss_MB: 1685.9 | child_rss_MB: 22497.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 08:04:20,044] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 08:04:20,044] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:22,987] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 08:05:22,988] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:45,997] INFO : Step 80000/500000 rank0 | Train Loss: 4.4951 | Train Loss Unweighted: 2.6166 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.4391 | Regression_cos_phi: 0.0063 | Regression_energy: 0.0700 | Regression_eta: 0.0044 | Regression_pt: 0.0682 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.2010 | Classification_binary: 1.0000 | Regression_cos_phi: 11.2052 | Regression_energy: 10.4247 | Regression_eta: 7.8836 | Regression_pt: 13.2302 | Regression_sin_phi: 10.8342 | Task Weighted Losses: Classification: 0.2470 | Classification_binary: 2.4391 | Regression_cos_phi: 0.0703 | Regression_energy: 0.7302 | Regression_eta: 0.0348 | Regression_pt: 0.9024 | Regression_sin_phi: 0.0714 | LR: 2.85e-04 | DataLoad Time: 0.0016s | Model Forward Time: 0.6685s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0029 | max_backward: 0.0404 | max_optimizer_step: 0.0705 | max_tensorboard_logging: 0.2682 | max_tensorboard_flush: 0.0003 | Memory: rss_MB: 2830.4 | uss_MB: 1686.7 | child_rss_MB: 22496.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 08:05:45,999] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 08:05:46,078] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 08:05:46,080] INFO : Running validation on rank0 +rank0 [2026-11-14 08:05:46,080] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 08:05:46,080] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 08:05:46,236] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 08:05:46,357] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 08:05:46,476] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 08:05:46,547] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 08:05:46,615] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 08:05:46,682] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 08:05:46,751] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 08:05:46,819] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 08:05:46,923] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 08:05:47,040] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 08:05:47,159] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 08:05:47,228] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 08:05:47,295] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 08:05:47,362] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 08:05:47,430] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 08:05:47,497] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 08:05:47,607] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 08:05:47,613] INFO : plotted pt_target histogram +rank0 [2026-11-14 08:05:47,615] INFO : plotted pt_pred histogram +rank0 [2026-11-14 08:05:47,617] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 08:05:47,619] INFO : plotted eta_target histogram +rank0 [2026-11-14 08:05:47,622] INFO : plotted eta_pred histogram +rank0 [2026-11-14 08:05:47,624] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 08:05:47,626] INFO : plotted sphi_target histogram +rank0 [2026-11-14 08:05:47,628] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 08:05:47,631] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 08:05:47,633] INFO : plotted cphi_target histogram +rank0 [2026-11-14 08:05:47,635] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 08:05:47,637] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 08:05:47,640] INFO : plotted energy_target histogram +rank0 [2026-11-14 08:05:47,642] INFO : plotted energy_pred histogram +rank0 [2026-11-14 08:05:47,644] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 08:05:49,174] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.4951 | Valid Loss=9.6425 | Train Loss Unweighted=2.6166 | Valid Loss Unweighted=4.3894 | Stale=3 | ETA=796.2m +rank0 [2026-11-14 08:05:49,174] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 08:05:49,637] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 08:05:49,651] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 08:05:49,705] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 08:05:50,235] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 08:05:50,511] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 08:05:50,761] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 08:05:51,018] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 08:05:51,290] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 08:05:51,547] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 08:05:51,804] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 08:05:52,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 08:05:52,332] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 08:05:52,585] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 08:05:52,842] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 08:05:53,104] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 08:05:53,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 08:05:53,643] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 08:05:53,900] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 08:05:54,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 08:05:54,198] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 08:05:54,207] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:05:54,785] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_25mm_seed12345_20261114_053324_446046/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:05:55,185] INFO : Plotted number of elements +rank0 [2026-11-14 08:05:56,528] INFO : Plotted jets +rank0 [2026-11-14 08:05:57,508] INFO : Plotted jet ratio +rank0 [2026-11-14 08:05:57,986] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 08:05:57,986] INFO : Training completed. Total time on device 0: 151.796min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..1f5fb7eea84ff2b381779033defbcae82ef9cf70 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052c198fb4b8ad66d6d31470ad3ef8f9b6f624b65c2bf3f3f7e139cf0be0ca92 +size 47063573 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-80000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-80000.pth new file mode 100644 index 0000000000000000000000000000000000000000..39a9b7d27d1016f40bfe27b7cc942fe2dedf32e4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-80000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:473cab85b43cae5f62ced8eecd473498fbbd86ee2e8648de157c86dfb0bd6644 +size 47101757 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..cb88adbf333c97c28dd4dc9c47dad4450c9c9fd7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 17.793594306049823, "train": {"Classification_binary": 3.605140447616577, "Classification": 0.03531859070062637, "Regression_pt": 0.17877963185310364, "Regression_eta": 0.009391970001161098, "Regression_sin_phi": 0.009136966429650784, "Regression_cos_phi": 0.008664065971970558, "Regression_energy": 0.1821678727865219, "Total": 8.452791213989258, "Total_unweighted": 4.0285995453596115}, "train_group": {}, "valid": {"Classification_binary": 3.9408516883850098, "Classification": 0.03644393011927605, "Regression_pt": 0.20578959584236145, "Regression_eta": 0.010972262360155582, "Regression_sin_phi": 0.011298692785203457, "Regression_cos_phi": 0.011822991073131561, "Regression_energy": 0.2028564065694809, "Total": 9.433839797973633, "Total_unweighted": 4.420035567134619, "metrics/particle/class_1/efficiency": 0.8854394555091858, "metrics/particle/class_1/pid_accuracy": 0.9808394908905029, "metrics/particle/class_2/efficiency": 0.20035046339035034, "metrics/particle/class_2/pid_accuracy": 0.26822158694267273, "metrics/particle/class_3/efficiency": 0.7099554538726807, "metrics/particle/class_3/pid_accuracy": 0.9766743779182434, "metrics/particle/class_4/efficiency": 0.8766233921051025, "metrics/particle/class_4/pid_accuracy": 0.04814814776182175, "metrics/particle/class_5/efficiency": 0.9709302186965942, "metrics/particle/class_5/pid_accuracy": 0.1796407252550125, "metrics/particle/count/bias_mean": -1.4520000219345093, "metrics/particle/count/mae": 4.947999954223633, "metrics/particle/count/prediction_mean": 94.36000061035156, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.07745085656642914, "metrics/particle/event/energy_response_mean": 0.9562041759490967, "metrics/particle/event/met_abs_error": 13.256671905517578, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07903298735618591, "metrics/particle/event/scalar_pt_response_mean": 0.9613778591156006, "metrics/particle/event/vector_pt_closure": 0.08941012620925903, "metrics/particle/matched/delta_eta_abs_mean": 0.021304897964000702, "metrics/particle/matched/delta_phi_abs_mean": 0.019770415499806404, "metrics/particle/matched/delta_r_mean": 0.03238649293780327, "metrics/particle/matched/energy_relative_abs_mean": 0.09543173015117645, "metrics/particle/matched/pid_accuracy": 0.9446150660514832, "metrics/particle/matched/pt_relative_abs_mean": 0.09110615402460098, "metrics/particle/matching/duplicate_fraction": 0.040610428899526596, "metrics/particle/matching/efficiency": 0.7613242864608765, "metrics/particle/matching/f1": 0.7671371102333069, "metrics/particle/matching/purity": 0.7730394005775452, "metrics/particle/matching/target_coverage_dr0p05": 0.6290652751922607, "metrics/particle/matching/target_coverage_dr0p10": 0.7922180891036987, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6798731088638306, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8198555707931519}, "step_train_time": 0.49068355560302734, "step_valid_time": 2.753893696004525, "step_total_time": 1088.918875694275, "elapsed_training_loop_seconds": 1088.918875694275, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9832034621185847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.21538096462139733, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1516, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1313, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1241, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8632478632478633, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8660949868073878, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8646690813302601, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.13390501319261214, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8159105851413544, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8186015831134564, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8172538689496214, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8632478632478633, "step_test_time": 4.851701361010782, "step_plot_time": 4.1886836549965665, "step_validation_cycle_time": 12.200702823000029} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..64c24a91d8fc7cccb97e4ce03a94030387492967 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 35.587188612099645, "train": {"Classification_binary": 3.8874244689941406, "Classification": 0.034221209585666656, "Regression_pt": 0.2329915165901184, "Regression_eta": 0.009653713554143906, "Regression_sin_phi": 0.010366130620241165, "Regression_cos_phi": 0.006599023938179016, "Regression_energy": 0.23060102760791779, "Total": 9.912080764770508, "Total_unweighted": 4.411857090890408}, "train_group": {}, "valid": {"Classification_binary": 3.7769386768341064, "Classification": 0.03448369354009628, "Regression_pt": 0.19640615582466125, "Regression_eta": 0.00991670973598957, "Regression_sin_phi": 0.010511312633752823, "Regression_cos_phi": 0.011058473959565163, "Regression_energy": 0.19320321083068848, "Total": 9.002007484436035, "Total_unweighted": 4.23251823335886, "metrics/particle/class_1/efficiency": 0.8871884346008301, "metrics/particle/class_1/pid_accuracy": 0.9811729788780212, "metrics/particle/class_2/efficiency": 0.21612149477005005, "metrics/particle/class_2/pid_accuracy": 0.36486485600471497, "metrics/particle/class_3/efficiency": 0.7091807126998901, "metrics/particle/class_3/pid_accuracy": 0.9778779149055481, "metrics/particle/class_4/efficiency": 0.8766233921051025, "metrics/particle/class_4/pid_accuracy": 0.08518518507480621, "metrics/particle/class_5/efficiency": 0.9709302186965942, "metrics/particle/class_5/pid_accuracy": 0.20359280705451965, "metrics/particle/count/bias_mean": -2.9719998836517334, "metrics/particle/count/mae": 5.283999919891357, "metrics/particle/count/prediction_mean": 92.83999633789062, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.06660409271717072, "metrics/particle/event/energy_response_mean": 1.0012071132659912, "metrics/particle/event/met_abs_error": 13.28987979888916, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06637746840715408, "metrics/particle/event/scalar_pt_response_mean": 1.0001740455627441, "metrics/particle/event/vector_pt_closure": 0.08156508952379227, "metrics/particle/matched/delta_eta_abs_mean": 0.018746821209788322, "metrics/particle/matched/delta_phi_abs_mean": 0.017714040353894234, "metrics/particle/matched/delta_r_mean": 0.028692444786429405, "metrics/particle/matched/energy_relative_abs_mean": 0.10724356770515442, "metrics/particle/matched/pid_accuracy": 0.9470314383506775, "metrics/particle/matched/pt_relative_abs_mean": 0.10682429373264313, "metrics/particle/matching/duplicate_fraction": 0.0405859537422657, "metrics/particle/matching/efficiency": 0.762952446937561, "metrics/particle/matching/f1": 0.7749719023704529, "metrics/particle/matching/purity": 0.7873761057853699, "metrics/particle/matching/target_coverage_dr0p05": 0.6720661520957947, "metrics/particle/matching/target_coverage_dr0p10": 0.8033649325370789, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6559094786643982, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.811798095703125}, "step_train_time": 0.5241866111755371, "step_valid_time": 2.99368721304927, "step_total_time": 2180.4384257793427, "elapsed_training_loop_seconds": 2180.4384257793427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0191518074562245, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.20883888339409462, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1528, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1343, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1281, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8829717291255753, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8789267015706806, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8809445719908167, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12107329842931937, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8422090729783037, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8383507853403142, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8402755001639882, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8829717291255753, "step_test_time": 4.567901831003837, "step_plot_time": 3.2262523300014436, "step_validation_cycle_time": 11.271726078004576} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..9398ff83f7515422d535f95f018f4f16592ecd49 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 53.380782918149464, "train": {"Classification_binary": 3.406646728515625, "Classification": 0.03109591081738472, "Regression_pt": 0.1831769347190857, "Regression_eta": 0.008780750446021557, "Regression_sin_phi": 0.012127937749028206, "Regression_cos_phi": 0.007652066648006439, "Regression_energy": 0.18741752207279205, "Total": 8.33365535736084, "Total_unweighted": 3.8368978509679437}, "train_group": {}, "valid": {"Classification_binary": 3.6668219566345215, "Classification": 0.033459220081567764, "Regression_pt": 0.18953898549079895, "Regression_eta": 0.0095176687464118, "Regression_sin_phi": 0.010142984800040722, "Regression_cos_phi": 0.010680609382689, "Regression_energy": 0.1867949515581131, "Total": 8.714468955993652, "Total_unweighted": 4.106956376694143, "metrics/particle/class_1/efficiency": 0.9024049043655396, "metrics/particle/class_1/pid_accuracy": 0.9793584942817688, "metrics/particle/class_2/efficiency": 0.22429905831813812, "metrics/particle/class_2/pid_accuracy": 0.3723958432674408, "metrics/particle/class_3/efficiency": 0.7469494342803955, "metrics/particle/class_3/pid_accuracy": 0.9791261553764343, "metrics/particle/class_4/efficiency": 0.8896104097366333, "metrics/particle/class_4/pid_accuracy": 0.11313868314027786, "metrics/particle/class_5/efficiency": 0.9709302186965942, "metrics/particle/class_5/pid_accuracy": 0.329341322183609, "metrics/particle/count/bias_mean": 1.8279999494552612, "metrics/particle/count/mae": 4.73199987411499, "metrics/particle/count/prediction_mean": 97.63999938964844, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.06798083335161209, "metrics/particle/event/energy_response_mean": 0.9534264206886292, "metrics/particle/event/met_abs_error": 11.730834007263184, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06769829243421555, "metrics/particle/event/scalar_pt_response_mean": 0.9524919390678406, "metrics/particle/event/vector_pt_closure": 0.07271554321050644, "metrics/particle/matched/delta_eta_abs_mean": 0.018042000010609627, "metrics/particle/matched/delta_phi_abs_mean": 0.018264232203364372, "metrics/particle/matched/delta_r_mean": 0.02856413833796978, "metrics/particle/matched/energy_relative_abs_mean": 0.09521225094795227, "metrics/particle/matched/pid_accuracy": 0.9485602378845215, "metrics/particle/matched/pt_relative_abs_mean": 0.09234072268009186, "metrics/particle/matching/duplicate_fraction": 0.03965587913990021, "metrics/particle/matching/efficiency": 0.7872500419616699, "metrics/particle/matching/f1": 0.7798110246658325, "metrics/particle/matching/purity": 0.7725112438201904, "metrics/particle/matching/target_coverage_dr0p05": 0.6839644312858582, "metrics/particle/matching/target_coverage_dr0p10": 0.8214002251625061, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6930238604545593, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8403957486152649}, "step_train_time": 0.5175652503967285, "step_valid_time": 3.002144630998373, "step_total_time": 3273.684479236603, "elapsed_training_loop_seconds": 3273.684479236603, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9770344703167116, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18502932303610664, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1524, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1347, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1304, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8856015779092702, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8838582677165354, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8847290640394089, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11614173228346457, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8573307034845496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8556430446194225, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8564860426929393, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8856015779092702, "step_test_time": 4.486936855013482, "step_plot_time": 3.2099512909771875, "step_validation_cycle_time": 11.200578907970339} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..ac8c9d62661a67cf1f5f67155be50479b10466ef --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 71.17437722419929, "train": {"Classification_binary": 3.278534173965454, "Classification": 0.023093199357390404, "Regression_pt": 0.17984092235565186, "Regression_eta": 0.00712558114901185, "Regression_sin_phi": 0.008535785600543022, "Regression_cos_phi": 0.009855162352323532, "Regression_energy": 0.1753503680229187, "Total": 7.9207305908203125, "Total_unweighted": 3.6823351928032935}, "train_group": {}, "valid": {"Classification_binary": 3.446821928024292, "Classification": 0.029821600764989853, "Regression_pt": 0.18012356758117676, "Regression_eta": 0.00943821668624878, "Regression_sin_phi": 0.009796025231480598, "Regression_cos_phi": 0.010381687432527542, "Regression_energy": 0.17852352559566498, "Total": 8.239261627197266, "Total_unweighted": 3.8649065513163805, "metrics/particle/class_1/efficiency": 0.9024049043655396, "metrics/particle/class_1/pid_accuracy": 0.977711021900177, "metrics/particle/class_2/efficiency": 0.26810747385025024, "metrics/particle/class_2/pid_accuracy": 0.501089334487915, "metrics/particle/class_3/efficiency": 0.752275824546814, "metrics/particle/class_3/pid_accuracy": 0.9823635220527649, "metrics/particle/class_4/efficiency": 0.8571428656578064, "metrics/particle/class_4/pid_accuracy": 0.2537878751754761, "metrics/particle/class_5/efficiency": 0.9593023061752319, "metrics/particle/class_5/pid_accuracy": 0.43030303716659546, "metrics/particle/count/bias_mean": 1.187999963760376, "metrics/particle/count/mae": 4.507999897003174, "metrics/particle/count/prediction_mean": 97.0, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.05738229304552078, "metrics/particle/event/energy_response_mean": 0.9742457866668701, "metrics/particle/event/met_abs_error": 9.364075660705566, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0578596256673336, "metrics/particle/event/scalar_pt_response_mean": 0.9790372252464294, "metrics/particle/event/vector_pt_closure": 0.060675282031297684, "metrics/particle/matched/delta_eta_abs_mean": 0.019190361723303795, "metrics/particle/matched/delta_phi_abs_mean": 0.017781618982553482, "metrics/particle/matched/delta_r_mean": 0.029129758477211, "metrics/particle/matched/energy_relative_abs_mean": 0.0967961996793747, "metrics/particle/matched/pid_accuracy": 0.9532542824745178, "metrics/particle/matched/pt_relative_abs_mean": 0.09346932172775269, "metrics/particle/matching/duplicate_fraction": 0.039463918656110764, "metrics/particle/matching/efficiency": 0.792176365852356, "metrics/particle/matching/f1": 0.787295401096344, "metrics/particle/matching/purity": 0.7824742197990417, "metrics/particle/matching/target_coverage_dr0p05": 0.6777439117431641, "metrics/particle/matching/target_coverage_dr0p10": 0.8242391347885132, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6951112747192383, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8475347757339478}, "step_train_time": 0.503241777420044, "step_valid_time": 2.896923344989773, "step_total_time": 4354.3768973350525, "elapsed_training_loop_seconds": 4354.3768973350525, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9872011922013189, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15337854400838824, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1498, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1352, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1330, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8888888888888888, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9025367156208278, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8956608148393508, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09746328437917223, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8744247205785667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8878504672897196, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8810864524677046, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8888888888888888, "step_test_time": 4.5449737689923495, "step_plot_time": 3.2389425999717787, "step_validation_cycle_time": 11.189421090006363} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_50000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_50000.json new file mode 100644 index 0000000000000000000000000000000000000000..7011ee4653f79a632a3348c4320bfb897a72f783 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_50000.json @@ -0,0 +1 @@ +{"epoch": 88.96797153024912, "train": {"Classification_binary": 3.1324660778045654, "Classification": 0.029385339468717575, "Regression_pt": 0.16353800892829895, "Regression_eta": 0.0077902949415147305, "Regression_sin_phi": 0.012136994861066341, "Regression_cos_phi": 0.012286907993257046, "Regression_energy": 0.16668467223644257, "Total": 7.617860317230225, "Total_unweighted": 3.5242882962338626}, "train_group": {}, "valid": {"Classification_binary": 3.3841516971588135, "Classification": 0.028785862028598785, "Regression_pt": 0.17855654656887054, "Regression_eta": 0.009414484724402428, "Regression_sin_phi": 0.009404154494404793, "Regression_cos_phi": 0.00989966094493866, "Regression_energy": 0.1772083193063736, "Total": 8.12134075164795, "Total_unweighted": 3.7974207252264023, "metrics/particle/class_1/efficiency": 0.9087013602256775, "metrics/particle/class_1/pid_accuracy": 0.9798864126205444, "metrics/particle/class_2/efficiency": 0.26460281014442444, "metrics/particle/class_2/pid_accuracy": 0.5121412873268127, "metrics/particle/class_3/efficiency": 0.7484021186828613, "metrics/particle/class_3/pid_accuracy": 0.9821428656578064, "metrics/particle/class_4/efficiency": 0.8571428656578064, "metrics/particle/class_4/pid_accuracy": 0.25757575035095215, "metrics/particle/class_5/efficiency": 0.9593023061752319, "metrics/particle/class_5/pid_accuracy": 0.4484848380088806, "metrics/particle/count/bias_mean": 0.7799999713897705, "metrics/particle/count/mae": 4.276000022888184, "metrics/particle/count/prediction_mean": 96.59200286865234, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.054332051426172256, "metrics/particle/event/energy_response_mean": 0.973855197429657, "metrics/particle/event/met_abs_error": 9.335541725158691, "metrics/particle/event/scalar_pt_relative_abs_error": 0.053313955664634705, "metrics/particle/event/scalar_pt_response_mean": 0.9769798517227173, "metrics/particle/event/vector_pt_closure": 0.05899417772889137, "metrics/particle/matched/delta_eta_abs_mean": 0.018920298665761948, "metrics/particle/matched/delta_phi_abs_mean": 0.017408007755875587, "metrics/particle/matched/delta_r_mean": 0.02863074652850628, "metrics/particle/matched/energy_relative_abs_mean": 0.09700687974691391, "metrics/particle/matched/pid_accuracy": 0.9550023674964905, "metrics/particle/matched/pt_relative_abs_mean": 0.0896894782781601, "metrics/particle/matching/duplicate_fraction": 0.03594500571489334, "metrics/particle/matching/efficiency": 0.7932618260383606, "metrics/particle/matching/f1": 0.7900459170341492, "metrics/particle/matching/purity": 0.7868560552597046, "metrics/particle/matching/target_coverage_dr0p05": 0.6792468428611755, "metrics/particle/matching/target_coverage_dr0p10": 0.8249906301498413, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7014987468719482, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8480357527732849}, "step_train_time": 0.512315034866333, "step_valid_time": 2.9490338820032775, "step_total_time": 5436.424929380417, "elapsed_training_loop_seconds": 5436.424929380417, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9828534175530712, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15056588869182463, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1507, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1368, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1346, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8994082840236687, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9077637690776377, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9035667107001321, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09223623092236231, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8849441157133465, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8931652289316523, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8890356671070013, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8994082840236687, "step_test_time": 4.505347742000595, "step_plot_time": 3.203311787045095, "step_validation_cycle_time": 11.131742492958438} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_60000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_60000.json new file mode 100644 index 0000000000000000000000000000000000000000..b40df6240b1114d9905946c09239ded52bb76ffa --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_60000.json @@ -0,0 +1 @@ +{"epoch": 106.76156583629893, "train": {"Classification_binary": 3.3403868675231934, "Classification": 0.025916660204529762, "Regression_pt": 0.12523992359638214, "Regression_eta": 0.01465551182627678, "Regression_sin_phi": 0.011783302761614323, "Regression_cos_phi": 0.01384060364216566, "Regression_energy": 0.1204937994480133, "Total": 6.887032508850098, "Total_unweighted": 3.6523166690021753}, "train_group": {}, "valid": {"Classification_binary": 3.3992655277252197, "Classification": 0.028598174452781677, "Regression_pt": 0.1938074827194214, "Regression_eta": 0.009485751390457153, "Regression_sin_phi": 0.0093031981959939, "Regression_cos_phi": 0.009599046781659126, "Regression_energy": 0.194083109498024, "Total": 8.500520706176758, "Total_unweighted": 3.844142290763557, "metrics/particle/class_1/efficiency": 0.9073021411895752, "metrics/particle/class_1/pid_accuracy": 0.9813011884689331, "metrics/particle/class_2/efficiency": 0.2517523467540741, "metrics/particle/class_2/pid_accuracy": 0.5359628796577454, "metrics/particle/class_3/efficiency": 0.7399767637252808, "metrics/particle/class_3/pid_accuracy": 0.9811543226242065, "metrics/particle/class_4/efficiency": 0.8571428656578064, "metrics/particle/class_4/pid_accuracy": 0.2537878751754761, "metrics/particle/class_5/efficiency": 0.9593023061752319, "metrics/particle/class_5/pid_accuracy": 0.4848484992980957, "metrics/particle/count/bias_mean": -1.8680000305175781, "metrics/particle/count/mae": 4.771999835968018, "metrics/particle/count/prediction_mean": 93.94400024414062, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.06325850635766983, "metrics/particle/event/energy_response_mean": 0.9683387875556946, "metrics/particle/event/met_abs_error": 9.818639755249023, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0638497918844223, "metrics/particle/event/scalar_pt_response_mean": 0.9712439179420471, "metrics/particle/event/vector_pt_closure": 0.06526908278465271, "metrics/particle/matched/delta_eta_abs_mean": 0.02037266455590725, "metrics/particle/matched/delta_phi_abs_mean": 0.018287602812051773, "metrics/particle/matched/delta_r_mean": 0.030439000576734543, "metrics/particle/matched/energy_relative_abs_mean": 0.0944201871752739, "metrics/particle/matched/pid_accuracy": 0.9565585851669312, "metrics/particle/matched/pt_relative_abs_mean": 0.09002312272787094, "metrics/particle/matching/duplicate_fraction": 0.03436089679598808, "metrics/particle/matching/efficiency": 0.7880432605743408, "metrics/particle/matching/f1": 0.7958009243011475, "metrics/particle/matching/purity": 0.8037128448486328, "metrics/particle/matching/target_coverage_dr0p05": 0.6617960333824158, "metrics/particle/matching/target_coverage_dr0p10": 0.8163069486618042, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.699327826499939, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8393520712852478}, "step_train_time": 0.5322849750518799, "step_valid_time": 2.9786031729890965, "step_total_time": 6531.0953867435455, "elapsed_training_loop_seconds": 6531.0953867435455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9888556276357338, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.14988145251482743, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1361, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1336, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8948060486522025, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9121983914209115, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9034185197477597, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08780160857908847, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8783694937541091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8954423592493298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8868237636906737, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8948060486522025, "step_test_time": 4.51610771700507, "step_plot_time": 3.2115882610087283, "step_validation_cycle_time": 11.10946036700625} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_70000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_70000.json new file mode 100644 index 0000000000000000000000000000000000000000..5303b8a19675817efb2d2a7e3bb9e27c9523536b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_70000.json @@ -0,0 +1 @@ +{"epoch": 124.55516014234875, "train": {"Classification_binary": 2.9051108360290527, "Classification": 0.020755965262651443, "Regression_pt": 0.11699727922677994, "Regression_eta": 0.006280742585659027, "Regression_sin_phi": 0.005686749704182148, "Regression_cos_phi": 0.006281559821218252, "Regression_energy": 0.11140377074480057, "Total": 5.975577354431152, "Total_unweighted": 3.172516903374344}, "train_group": {}, "valid": {"Classification_binary": 3.483870029449463, "Classification": 0.02938203327357769, "Regression_pt": 0.22286801040172577, "Regression_eta": 0.009342766366899014, "Regression_sin_phi": 0.008830789476633072, "Regression_cos_phi": 0.009494458325207233, "Regression_energy": 0.22361041605472565, "Total": 9.264713287353516, "Total_unweighted": 3.9873985033482313, "metrics/particle/class_1/efficiency": 0.907651960849762, "metrics/particle/class_1/pid_accuracy": 0.9773581027984619, "metrics/particle/class_2/efficiency": 0.2850467264652252, "metrics/particle/class_2/pid_accuracy": 0.5901639461517334, "metrics/particle/class_3/efficiency": 0.7294209003448486, "metrics/particle/class_3/pid_accuracy": 0.9761019945144653, "metrics/particle/class_4/efficiency": 0.8733766078948975, "metrics/particle/class_4/pid_accuracy": 0.2565055787563324, "metrics/particle/class_5/efficiency": 0.9534883499145508, "metrics/particle/class_5/pid_accuracy": 0.5060975551605225, "metrics/particle/count/bias_mean": -1.2319999933242798, "metrics/particle/count/mae": 4.736000061035156, "metrics/particle/count/prediction_mean": 94.58000183105469, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.057582128793001175, "metrics/particle/event/energy_response_mean": 0.9766238331794739, "metrics/particle/event/met_abs_error": 9.937150001525879, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05738724395632744, "metrics/particle/event/scalar_pt_response_mean": 0.9821478724479675, "metrics/particle/event/vector_pt_closure": 0.06411401182413101, "metrics/particle/matched/delta_eta_abs_mean": 0.01951383426785469, "metrics/particle/matched/delta_phi_abs_mean": 0.01755150593817234, "metrics/particle/matched/delta_r_mean": 0.029232099652290344, "metrics/particle/matched/energy_relative_abs_mean": 0.0958293229341507, "metrics/particle/matched/pid_accuracy": 0.9524214267730713, "metrics/particle/matched/pt_relative_abs_mean": 0.09099502116441727, "metrics/particle/matching/duplicate_fraction": 0.038316767662763596, "metrics/particle/matching/efficiency": 0.7862063050270081, "metrics/particle/matching/f1": 0.791293740272522, "metrics/particle/matching/purity": 0.7964474558830261, "metrics/particle/matching/target_coverage_dr0p05": 0.6628397107124329, "metrics/particle/matching/target_coverage_dr0p10": 0.8160147070884705, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6960297226905823, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8387675881385803}, "step_train_time": 0.5141439437866211, "step_valid_time": 2.9997058720327914, "step_total_time": 7627.162432909012, "elapsed_training_loop_seconds": 7627.162432909012, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9895346879265052, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15398999755102094, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1513, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1364, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1336, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8967784352399737, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9015201586252478, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8991430454845089, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09847984137475214, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8783694937541091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8830138797091871, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8806855636123929, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8967784352399737, "step_test_time": 4.588098368025385, "step_plot_time": 3.2193169240490533, "step_validation_cycle_time": 11.236869555024896} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_80000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_80000.json new file mode 100644 index 0000000000000000000000000000000000000000..72043c90f823637ea2e30d6bd0db60e9e539601c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/history/step_80000.json @@ -0,0 +1 @@ +{"epoch": 142.34875444839858, "train": {"Classification_binary": 2.4148073196411133, "Classification": 0.019173048436641693, "Regression_pt": 0.06682486832141876, "Regression_eta": 0.011286995373666286, "Regression_sin_phi": 0.007023946847766638, "Regression_cos_phi": 0.006860130932182074, "Regression_energy": 0.06433726102113724, "Total": 4.397950649261475, "Total_unweighted": 2.590313570573926}, "train_group": {}, "valid": {"Classification_binary": 3.865152359008789, "Classification": 0.031117411330342293, "Regression_pt": 0.24588817358016968, "Regression_eta": 0.009213253855705261, "Regression_sin_phi": 0.008833680301904678, "Regression_cos_phi": 0.009525440633296967, "Regression_energy": 0.24531081318855286, "Total": 10.184803009033203, "Total_unweighted": 4.415041131898761, "metrics/particle/class_1/efficiency": 0.9049409627914429, "metrics/particle/class_1/pid_accuracy": 0.9774835705757141, "metrics/particle/class_2/efficiency": 0.2844626307487488, "metrics/particle/class_2/pid_accuracy": 0.5667350888252258, "metrics/particle/class_3/efficiency": 0.7206081748008728, "metrics/particle/class_3/pid_accuracy": 0.9744657874107361, "metrics/particle/class_4/efficiency": 0.8668830990791321, "metrics/particle/class_4/pid_accuracy": 0.23595505952835083, "metrics/particle/class_5/efficiency": 0.9418604373931885, "metrics/particle/class_5/pid_accuracy": 0.4197530746459961, "metrics/particle/count/bias_mean": -0.46799999475479126, "metrics/particle/count/mae": 5.099999904632568, "metrics/particle/count/prediction_mean": 95.34400177001953, "metrics/particle/count/target_mean": 95.81199645996094, "metrics/particle/event/energy_relative_abs_error": 0.06199502944946289, "metrics/particle/event/energy_response_mean": 0.9913246035575867, "metrics/particle/event/met_abs_error": 10.101875305175781, "metrics/particle/event/scalar_pt_relative_abs_error": 0.06440469622612, "metrics/particle/event/scalar_pt_response_mean": 0.9950820207595825, "metrics/particle/event/vector_pt_closure": 0.06916972249746323, "metrics/particle/matched/delta_eta_abs_mean": 0.018973907455801964, "metrics/particle/matched/delta_phi_abs_mean": 0.01736600324511528, "metrics/particle/matched/delta_r_mean": 0.02866252139210701, "metrics/particle/matched/energy_relative_abs_mean": 0.09892064332962036, "metrics/particle/matched/pid_accuracy": 0.9501737356185913, "metrics/particle/matched/pt_relative_abs_mean": 0.09001778066158295, "metrics/particle/matching/duplicate_fraction": 0.037674106657505035, "metrics/particle/matching/efficiency": 0.7809042930603027, "metrics/particle/matching/f1": 0.7828161120414734, "metrics/particle/matching/purity": 0.7847373485565186, "metrics/particle/matching/target_coverage_dr0p05": 0.6572454571723938, "metrics/particle/matching/target_coverage_dr0p10": 0.81062912940979, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6902684569358826, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8350102305412292}, "step_train_time": 0.5718355178833008, "step_valid_time": 2.975391371990554, "step_total_time": 8723.209316968918, "elapsed_training_loop_seconds": 8723.209316968918, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004700456206412, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.15215626487248146, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1521, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1519, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1371, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1341, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9013806706114399, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9025674786043449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9019736842105263, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09743252139565503, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8816568047337278, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8828176431863067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8822368421052632, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9013806706114399, "step_test_time": 4.53468374197837, "step_plot_time": 3.2040974409901537, "step_validation_cycle_time": 11.152606543037109} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/hyperparameters.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/hyperparameters.json new file mode 100644 index 0000000000000000000000000000000000000000..bbf3dac4dfe106bf324d5bfd951c12a654dcea8b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/hyperparameters.json @@ -0,0 +1 @@ +{"num_mlpf_params": 3910930, "dataset": "cld", "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_30mm/tfds", "model": {"type": "attention", "input_encoding": "split", "learned_representation_mode": "last", "pt_mode": "direct-elemtype-split", "eta_mode": "linear", "sin_phi_mode": "linear", "cos_phi_mode": "linear", "energy_mode": "direct-elemtype-split", "trainable": "all", "task_queries": true, "output_mode": "elementwise", "binary_classification_focal_gamma": null, "backbone": {"mode": "shared", "num_convs": 6, "num_tracker_layers": null, "num_calo_layers": null, "num_common_layers": null}, "hit_feature_engineering": {"enabled": true, "geometry": true, "tracker_neighborhood": false, "tracker_momentum": false, "tracker_doublets": false, "calorimeter_neighborhood": true}, "set_decoder": null, "input_normalization": true, "feature_bag": true, "gnnlsh": null, "attention": {"num_convs": 6, "dropout_ff": 0.0, "activation": "elu", "layernorm": true, "num_heads": 16, "head_dim": 16, "attention_type": "flash", "dropout_conv_reg_mha": 0.0, "dropout_conv_reg_ff": 0.0, "dropout_conv_id_mha": 0.0, "dropout_conv_id_ff": 0.0, "use_pre_layernorm": true, "use_jagged_attention": false, "use_flash_attn_varlen": false, "export_onnx_fused": false, "save_attention": false}, "litept": null, "hept": null, "heptv2": null}, "conv_type": "attention", "batch_size": 1, "hyperparameters": {"batch_size": 1, "lr": 0.0005}, "backend": "pytorch", "threads": 16, "gpu_type": "l40", "mem_per_gpu_mb": 40000, "slurm_partition": "gpu", "slurm_runtime": "168h", "input_dim": 19, "num_classes": 6, "elemtypes_nonzero": [1, 2], "seed": 12345, "num_steps": 500000, "patience": 3, "checkpoint_freq": 2000, "val_freq": 10000, "num_workers": 8, "prefetch_factor": 2, "max_open_readers": 64, "gpu_batch_multiplier": 16, "dtype": "bfloat16", "lr": 0.0005, "weight_decay": 0.01, "optimizer": "lamb", "lr_schedule": "onecycle", "lr_schedule_config": {"onecycle": {"pct_start": 0.3}}, "task_loss_weights": {"calibration_steps": 100, "epsilon": 1e-08, "min_weight": 0.01, "max_weight": 1000.0}, "lr_finder": {"start_lr": 1e-07, "end_lr": 0.1, "num_steps": 1000, "smoothing": 0.98, "divergence_factor": 4.0}, "pad_to_multiple_elements": 128, "validation_diagnostics_batches": 0, "validation_linear_probe": false, "validation_linear_probe_max_events": 16, "validation_particle_metrics": true, "train": true, "lr_find": false, "test": true, "compile": false, "make_plots": true, "sort_data": false, "sampler_mode": "interleaved-shards", "load": null, "relaxed_load": false, "load_weights_only": false, "reset_task_loss_weights": false, "sampler_from_scratch": false, "comet": false, "comet_offline": false, "comet_name": "particleflow", "tensorboard_step_freq": 1000, "benchmark": {"enabled": true, "sample_interval_seconds": 1.0, "warmup_steps": 100, "gpu_timing_interval": 0}, "raytune": {"local_dir": null, "sched": "asha", "search_alg": null, "default_metric": "val_loss", "default_mode": "min", "asha": {"max_t": 200, "reduction_factor": 4, "brackets": 1, "grace_period": 10}, "hyperband": {"max_t": 200, "reduction_factor": 4}, "hyperopt": {"n_random_steps": 10}, "nevergrad": {"n_random_steps": 10}}, "train_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "valid_dataset": {"cld": {"physical": {"batch_size": 1, "weight": null, "samples": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": null}}}}}, "test_dataset": {"cld_edm_ttbar_pf": {"version": "3.3.0", "splits": ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"], "batch_size": 1}}, "enabled_test_datasets": ["cld_edm_ttbar_pf"], "ntrain": null, "nvalid": 256, "ntest": 256, "data_config": null, "gpus": 1} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/model_kwargs.pkl b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/model_kwargs.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e7d6520d19f5c5795426ac74a7119effb2df0bc5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/model_kwargs.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b7127b71f3d39f492cd6e7f532695ee840e872bb99a16f4ba38d974eb8004af +size 4394 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/particleflow_spec.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/particleflow_spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5a1927e34d819b3c881ddb2cf44f6a069b608b4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/particleflow_spec.yaml @@ -0,0 +1,3198 @@ +models: + defaults: + backend: pytorch + checkpoint_freq: 10000 + comet: false + dtype: bfloat16 + gpu_type: l40 + gpus: 1 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: &id001 + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: &id002 + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1.0e3 + min_weight: 0.01 + tensorboard_step_freq: 100 + threads: 16 + val_freq: 10000 + pyg-cld-hits-heptv2-sector-set-v1: + architecture: + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + width: 256 + input_encoding: split + output_mode: set + set_decoder: + attention_mode: sectorized + local_attention_radius: 0.4 + num_heads: 8 + num_layers: 2 + num_sectors: 32 + num_slots: 256 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + use_aggregate_anchors: true + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id007 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id003 + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id004 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id005 + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: cld_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-cld-hits-heptv2-set-v1: + architecture: + backbone: + mode: shared + num_convs: 6 + binary_classification_focal_gamma: 2.0 + heptv2: + block_size: 128 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 6 + num_heads: 16 + width: 256 + hit_feature_engineering: + tracker_neighborhood: true + input_encoding: split + output_mode: set + set_decoder: + aggregate_grid_size: 0.1 + attention_mode: global-flash + auxiliary_loss_weight: 0.25 + cardinality_loss_weight: 0.05 + event_consistency_loss_weight: 0.0 + hit_grouping: true + hit_grouping_chunk_size: 1024 + hit_grouping_detector_balanced: false + hit_grouping_dim: 32 + hit_grouping_loss_weight: 0.1 + local_attention_radius: 0.4 + matcher: + dr_scale: 0.1 + energy: 0.0 + geometry: 2.0 + log_energy_scale: 0.6931471805599453 + log_pt_scale: 0.6931471805599453 + pid: 1.0 + presence: 1.0 + pt: 1.0 + no_object_weight: 1.0 + num_heads: 8 + num_layers: 4 + num_sectors: 32 + num_slots: 256 + presence_threshold: 0.5 + proposal_grid_size: 0.1 + proposal_mode: grid-diverse + query_init: input-conditioned + sector_neighbors: 1 + tracker_fit_correction: false + tracker_group_magnetic_field: 2.0 + tracker_group_readout: false + tracker_query_fraction: 0.6 + use_aggregate_anchors: true + task_queries: false + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-set-v1: + architecture: + attention: &id006 + head_dim: 16 + num_convs: 3 + num_heads: 16 + use_jagged_attention: true + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-hits-v1: + architecture: + attention: *id006 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: *id007 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id003 + threads: 16 + train_datasets: *id004 + val_freq: 10000 + validation_datasets: *id005 + pyg-cld-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cldcliccolliderml-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cld + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: 1.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + weight: 1.0 + val_freq: 10000 + validation_datasets: + cld: + batch_size: 1 + samples: + - name: cld_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: cld_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + clic: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + colliderml: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-clic-hits-set-v1: + architecture: + attention: &id008 + head_dim: 16 + num_convs: 3 + num_heads: 16 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 256 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id009 + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + threads: 16 + train_datasets: &id010 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + val_freq: 10000 + validation_datasets: &id011 + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_ww_fullhad_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + - name: clic_edm_qq_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.2 + pyg-clic-hits-v1: + architecture: + attention: *id008 + binary_classification_focal_gamma: 2.0 + input_encoding: split + output_mode: elementwise + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic_hits + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id009 + threads: 16 + train_datasets: *id010 + val_freq: 10000 + validation_datasets: *id011 + pyg-clic-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + cluster_hash_at_centroid: true + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: clic + dtype: bfloat16 + gpu_batch_multiplier: 1024 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: clic_edm_ttbar_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_ww_fullhad_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + - name: clic_edm_qq_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + pyg-cms-v1: + architecture: + attention: + head_dim: 32 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: cms + dtype: bfloat16 + gpu_batch_multiplier: 4 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0004 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + threads: 16 + train_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + val_freq: 10000 + validation_datasets: + physical_nopu: + batch_size: 16 + samples: + - name: cms_pf_ttbar_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt_nopu + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + physical_pu: + batch_size: 1 + samples: + - name: cms_pf_ttbar + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_qcd + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + - name: cms_pf_ztt + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.2.0 + pyg-colliderml-hits-set-v1: + architecture: + attention: &id012 + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + output_mode: set + set_decoder: + dropout: 0.0 + ffn_multiplier: 4.0 + num_heads: 8 + num_layers: 2 + num_slots: 512 + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id013 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 128 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: &id014 + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + threads: 16 + train_datasets: &id015 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + val_freq: 10000 + validation_datasets: &id016 + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_hits + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.3.0 + pyg-colliderml-hits-v1: + architecture: + attention: *id012 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml_hits + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id013 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: *id014 + threads: 16 + train_datasets: *id015 + val_freq: 10000 + validation_datasets: *id016 + pyg-colliderml-pu0pu200-groups-v1: + architecture: &id017 + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: &id018 + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + pu0: + batch_size: 12 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pu200: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-pu200-v1: + architecture: + attention: + attention_type: flash + head_dim: 16 + num_convs: 3 + num_heads: 16 + heptv2: + track_hash_at_calo: true + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_pu200_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-colliderml-v1: + architecture: *id017 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: colliderml + dtype: bfloat16 + gpu_batch_multiplier: 8 + gpu_type: l40 + gpus: 1 + hyperparameters: *id018 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: colliderml_ttbar_nopu_pf + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 1.5.0 + pyg-idea-pipeline-v1: + architecture: + heptv2: + block_size: 100 + dropout_ff: 0.0 + embedding_dim: 256 + num_convs: 3 + num_heads: 16 + track_hash_at_calo: true + width: 256 + input_encoding: split + type: heptv2 + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: idea + dtype: bfloat16 + gpu_batch_multiplier: 16 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0005 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + pad_to_multiple_elements: 100 + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: idea_edm_qq_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ttbar_pf + splits: + - '1' + version: 0.2.1 + - name: idea_edm_ww_fullhad_pf + splits: + - '1' + version: 0.2.1 + pyg-maia-v1: + architecture: + attention: + head_dim: 16 + num_convs: 3 + num_heads: 16 + input_encoding: split + type: attention + backend: pytorch + checkpoint_freq: 10000 + comet: false + dataset: maia + dtype: bfloat16 + gpu_batch_multiplier: 1 + gpu_type: l40 + gpus: 1 + hyperparameters: + batch_size: 1 + lr: 0.0001 + load: null + lr_schedule: cosinedecay + mem_per_gpu_mb: 40000 + ntest: 100 + ntrain: null + num_steps: 100000 + num_workers: 2 + nvalid: null + optimizer: lamb + patience: 10000 + prefetch_factor: 2 + raytune: *id001 + seed: 12345 + slurm_partition: ${project.gpu_partition} + slurm_runtime: 168h + sort_data: false + task_loss_weights: *id002 + tensorboard_step_freq: 100 + test_datasets: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + threads: 16 + train_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 + val_freq: 10000 + validation_datasets: + physical: + batch_size: 1 + samples: + - name: maia_edm_ttbar_pf + splits: + - '10' + version: 1.1.0 +productions: + cld: + comment: CLD production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 410010 + z_qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_qq_ecm91 + seed_range: + - 600000 + - 610010 + z_tautau: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 710010 + zh_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm240 + seed_range: + - 200000 + - 200110 + zh_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZH_Htautau_ecm365 + seed_range: + - 200000 + - 200110 + zz: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_ecm365 + seed_range: + - 100000 + - 105010 + zz_tautau_240: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm240 + seed_range: + - 100000 + - 100110 + zz_tautau_365: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/cld/run_sim.sh + process_name: p8_ee_ZZ_tautau_ecm365 + seed_range: + - 100000 + - 110010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/cld_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + zz: + builder_path: mlpf/heptfds/cld_pf_edm4hep/zz + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/v1.2.5_key4hep_2025-05-29 + cld_clic_colliderml: + comment: Joint CLD+CLIC+ColliderML pu0 tracks/clusters + model: pyg-cldcliccolliderml-v1 + source_productions: + cld: cld + clic: clic + colliderml: colliderml + type: key4hep + cld_ecal_pitch_1p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 1.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_1p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id019 + gen: 8000 + post: 8000 + tfds: 8000 + model: pyg-cld-v1 + postprocessing: &id020 + args: {} + script: mlpf/data/key4hep/postprocessing.py + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: &id021 + z_tautau: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_Z_tautau_ecm91 + seed_range: + - 700000 + - 700100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id022 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id023 + z_tautau: + builder_path: mlpf/heptfds/cld_pf_edm4hep/z_tautau + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_1p5mm + cld_ecal_pitch_2: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2mm + cld_ecal_pitch_2p5: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 2.5 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_2p5mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_2p5mm + cld_ecal_pitch_3: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 3 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_3mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_3mm + cld_ecal_pitch_4: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 4 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_4mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_4mm + cld_ecal_pitch_5p1: + cms_energy: 91 + comment: CLD ECAL pitch scan, Z->tautau at 91 GeV, 5.1 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/ecal_5p1mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id019 + model: pyg-cld-v1 + postprocessing: *id020 + pythia_card: ${project.paths.project_root}/scripts/cld_ecal_readout/cards/p8_ee_Z_tautau_ecm91_scan.cmd + samples: *id021 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id022 + tfds_mapping: *id023 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/ecal_5p1mm + cld_hcal_pitch_10: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 10 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_10mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: &id024 + gen: 12000 + post: 16000 + tfds: 16000 + model: pyg-cld-v1 + postprocessing: &id025 + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: &id026 + ttbar: + events_per_job: 100 + gen_script: scripts/cld_ecal_readout/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: &id027 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: &id028 + ttbar: + builder_path: mlpf/heptfds/cld_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_10mm + cld_hcal_pitch_12: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 12 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_12mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_12mm + cld_hcal_pitch_15: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 15 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_15mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_15mm + cld_hcal_pitch_20: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 20 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_20mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_20mm + cld_hcal_pitch_25: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 25 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_25mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_25mm + cld_hcal_pitch_30: + cms_energy: 365 + comment: CLD HCAL pitch scan, ttbar at 365 GeV, 30 mm + compact_file: ${project.paths.project_root}/mlpf/data/key4hep/geometry/granularity_scan/hcal_30mm/CLD_o2_v07.xml + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/cld/CLDConfig + dataset_only: true + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: *id024 + model: pyg-cld-v1 + postprocessing: *id025 + samples: *id026 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: *id027 + tfds_mapping: *id028 + type: key4hep + workspace_dir: ${project.paths.storage_root}/cld/granularity_scan_2025-05-29/hcal_30mm + clic: + comment: CLIC production with Key4Hep + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/clic + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 4000 + post: 2000 + tfds: 16000 + model: pyg-clic-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_qq_ecm380 + seed_range: + - 500000 + - 510010 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_ttbar_ecm380 + seed_range: + - 300000 + - 310010 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/clic/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm380 + seed_range: + - 400000 + - 410010 + slurm_partition: main + slurm_runtime: 48h + tfds_hit_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep_hits/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.2 + tfds_mapping: + qq: + builder_path: mlpf/heptfds/clic_pf_edm4hep/qq + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ttbar: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + ww_fullhad: + builder_path: mlpf/heptfds/clic_pf_edm4hep/ww_fullhad + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.3.0 + type: key4hep + workspace_dir: ${project.paths.storage_root}/clic/v1.2.5_key4hep_2025-05-29 + cms_run3: + comment: Main production with CMSSW_15_0_5 + config_dir: ${project.paths.project_root} + environment: + cmssw_dir: ${project.cmssw_dir} + cmssw_release: CMSSW_15_0_5 + scram_arch: el8_amd64_gcc12 + gen_container: /cvmfs/singularity.opensciencegrid.org/cmssw/cms:rhel8-x86_64 + memory: + gen: 6000 + post: 4000 + tfds: 16000 + train: 16000 + val: 8000 + model: pyg-cms-v1 + postprocessing: + args: + num_events: -1 + save_full_graph: false + script: mlpf/data/cms/postprocessing2.py + samples: + qcd_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1000000 + - 1010010 + qcd_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 300000 + - 310010 + qcd_val: + copy_step2: true + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75_val + process_name: QCDForPF_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 400000 + - 401010 + ttbar_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 800000 + - 810010 + ttbar_pu: + events_per_job: 50 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: TTbar_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 100000 + - 110010 + ztt_nopu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: nopu + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: nopu + seed_range: + - 1100000 + - 1110010 + ztt_pu: + events_per_job: 100 + gen_script: mlpf/data/cms/genjob_pu.sh + output_subdir: pu55to75 + process_name: ZTT_All_hadronic_13p6TeV_TuneCUETP8M1_cfi + pu_type: pu55to75 + seed_range: + - 200000 + - 210010 + slurm_partition: ${project.cpu_partition} + slurm_runtime: 48h + tfds_mapping: + qcd_nopu: + builder_path: mlpf/heptfds/cms_pf/qcd_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + qcd_pu: + builder_path: mlpf/heptfds/cms_pf/qcd + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_nopu: + builder_path: mlpf/heptfds/cms_pf/ttbar_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ttbar_pu: + builder_path: mlpf/heptfds/cms_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_nopu: + builder_path: mlpf/heptfds/cms_pf/ztt_nopu + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + ztt_pu: + builder_path: mlpf/heptfds/cms_pf/ztt + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 3.2.0 + type: cms + validation: + job_types: + - pf + - mlpf + threads: 8 + use_cuda: false + validation_data: + job_types: + - pf + - mlpf + samples: + jetmet0: + input_filelist: data/cms/run3/${project.site_name}/jetmet0.txt + output_subdir: Run2024C + seed_range: + - 1 + - 100 + threads: 8 + use_cuda: false + workspace_dir: ${project.paths.storage_root}/cms/20260204_cmssw_15_0_5_117d32 + colliderml: + comment: ColliderML Release 1 ttbar (no pileup), clustered view (tracks + our + spatial clusters) + download_root: /mnt/ceph/users/ewulff/data/colliderml/CERN__ColliderML-Release-1 + memory: + post: 32000 + tfds: 32000 + postprocessing: + args: {} + script: mlpf/data/colliderml/postprocessing.py + samples: + ttbar_pu0: + source_dir: ${productions.colliderml.download_root} + ttbar_pu200: + source_dir: ${productions.colliderml.download_root} + source_revision: 64c3d2f112df3d5d20979d22da7cfdff13e10c4b + tfds_hit_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_hits/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_hits/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.3.0 + tfds_mapping: + ttbar_pu0: + builder_path: mlpf/heptfds/colliderml_pf/ttbar + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + ttbar_pu200: + builder_path: mlpf/heptfds/colliderml_pf/ttbar_pu200 + config_ids: + - 1 + - 2 + - 3 + - 4 + - 5 + - 6 + - 7 + - 8 + - 9 + - 10 + version: 1.5.0 + type: colliderml + workspace_dir: ${productions.colliderml.workspace_root}/v1_release1_64c3d2f1 + workspace_root: /mnt/ceph/users/ewulff/mlpf/colliderml + idea: + comment: IDEA_o1_v03 pipeline validation with truth-seeded proxy tracks + config_dir: ${project.paths.project_root}/mlpf/data/key4hep/gen/idea + environment: + progress_interval: 60 + gen_container: /cvmfs/unpacked.cern.ch/gitlab-registry.cern.ch/key4hep/k4-deploy/alma9:latest + memory: + gen: 16000 + post: 4000 + tfds: 8000 + model: pyg-idea-pipeline-v1 + postprocessing: + args: {} + script: mlpf/data/key4hep/postprocessing.py + samples: + gun_e_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_e_10gev + seed_range: + - 424242 + - 424342 + gun_mu_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_mu_10gev + seed_range: + - 600000 + - 600100 + gun_pi_10gev: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: gun_pi_10gev + seed_range: + - 700000 + - 700100 + qq: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_qq_ecm365 + seed_range: + - 500000 + - 500100 + ttbar: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_ttbar_ecm365 + seed_range: + - 300000 + - 300100 + ww_fullhad: + events_per_job: 100 + gen_script: mlpf/data/key4hep/gen/idea/run_sim.sh + process_name: p8_ee_WW_fullhad_ecm365 + seed_range: + - 400000 + - 400100 + slurm_partition: main + slurm_runtime: 48h + tfds_mapping: + qq: + builder_path: mlpf/heptfds/idea_pf_edm4hep/qq + config_ids: + - 1 + version: 0.2.1 + ttbar: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ttbar + config_ids: + - 1 + version: 0.2.1 + ww_fullhad: + builder_path: mlpf/heptfds/idea_pf_edm4hep/ww_fullhad + config_ids: + - 1 + version: 0.2.1 + type: key4hep + workspace_dir: ${project.paths.storage_root}/idea/IDEA_o1_v03_fccconfig_a05a3a9 + maia: + comment: Local MAIA ttbar EDM4hep validation sample + config_dir: ${project.paths.project_root} + model: pyg-maia-v1 + type: maia + workspace_dir: ${project.paths.project_root}/local_test_data/maia_smoke +project: + bind_mounts: &id029 + - /local + - /cvmfs + - /scratch/local + - /scratch/persistent + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + description: Machine Learning for Particle Flow Reconstruction + executor: slurm + gpu_partition: gpu + name: particleflow + paths: &id030 + project_root: /home/joosep/particleflow + scratch_root: /scratch/local/joosep + storage_root: /local/joosep/mlpf + tmpdir: /scratch/local/joosep/tmp + site_name: tallinn + sites: + local: + bind_mounts: + - /tmp + - /mnt/work + cmssw_dir: /home/joosep/particleflow/CMSSW_15_0_5 + container: https://jpata.web.cern.ch/jpata/particleflow-29042026-de861d.sif + cpu_partition: local + executor: local + gpu_partition: local + paths: + project_root: /home/joosep/particleflow + scratch_root: /tmp/particleflow + storage_root: /mnt/work/mlpf/ + tmpdir: /tmp/particleflow/tmp + site_name: local + lxplus: + bind_mounts: + - /afs + - /cvmfs + - /eos/user/j/jpata + - /etc/pki/ca-trust + - /etc/tnsnames.ora + - /run + cmssw_dir: /cvmfs/cms.cern.ch + container: /eos/user/j/jpata/www/particleflow-20260330-cd2f54.sif + cpu_partition: tomorrow + executor: condor + gpu_partition: tomorrow + paths: + project_root: /afs/cern.ch/work/j/jpata/particleflow + scratch_root: /tmp + storage_root: /eos/user/j/jpata/mlpf + tmpdir: /tmp + site_name: lxplus + tallinn: + bind_mounts: *id029 + cmssw_dir: /scratch/persistent/joosep/CMSSW_15_0_5 + container: /scratch/persistent/joosep/singularity/particleflow-29042026-de861d.sif + cpu_partition: main + executor: slurm + gpu_partition: gpu + paths: *id030 + site_name: tallinn + slurm_account: hepusers + slurm_account: hepusers diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf new file mode 100644 index 0000000000000000000000000000000000000000..2052684c90507059b5b0dd1e704385968dd4ab22 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_eta.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png new file mode 100644 index 0000000000000000000000000000000000000000..e2c4998e59a67be434262e2379172048d74149d2 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_eta.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fd03c630b43af000f5bf406364fbf48894c75887 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png new file mode 100644 index 0000000000000000000000000000000000000000..47fcb9fa01851b8d27e196fe95ce0afef5fa62eb Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf new file mode 100644 index 0000000000000000000000000000000000000000..16927a77f44d46e5f55e25db080df3c873344c14 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png new file mode 100644 index 0000000000000000000000000000000000000000..97e42b6c1e5cd8d7510f8b85043e2877a3fa8c65 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_pt_log.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf new file mode 100644 index 0000000000000000000000000000000000000000..c9aad7977289af1a07a37a5487004347c47ad467 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res.png new file mode 100644 index 0000000000000000000000000000000000000000..859e053e2fa76552cf634336887c741f2dbf1d3b Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8be348fc99d2f65cc6713451e76e8aee1ce78523 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png new file mode 100644 index 0000000000000000000000000000000000000000..ca9666ec35afb8a303c9c6d97722f9c3e8a6806d Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/jet_res_vstarget.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf new file mode 100644 index 0000000000000000000000000000000000000000..e44fc1b7cb89eb63d527e000dbfdba5a6a4b16ba Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/num_elements.pdf differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/num_elements.png b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/num_elements.png new file mode 100644 index 0000000000000000000000000000000000000000..38bed32c6746955b5702eda81aef210fa21ac5e4 Binary files /dev/null and b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/plots_step_80000/cld_edm_ttbar_pf/num_elements.png differ diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/runs/train/events.out.tfevents.1794654475.workergpu164.1994251.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/runs/train/events.out.tfevents.1794654475.workergpu164.1994251.0 new file mode 100644 index 0000000000000000000000000000000000000000..960623f8f38b06fa0dc63ca96909e6a196fd55ea --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/runs/train/events.out.tfevents.1794654475.workergpu164.1994251.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b132c97ad0b1c45fefc45e22d5fd3842ed491b9a25bf2277510ba1f5e227fdb7 +size 2953648 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/runs/valid/events.out.tfevents.1794654475.workergpu164.1994251.1 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/runs/valid/events.out.tfevents.1794654475.workergpu164.1994251.1 new file mode 100644 index 0000000000000000000000000000000000000000..8d58a0e7f3c2f4d9aab5b9563cac298f99f0d90c --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/runs/valid/events.out.tfevents.1794654475.workergpu164.1994251.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fa0c44fcdd49d71cd032526207f1f06b573d10751f6d10f01a97ae9e3ea92dd +size 4013850 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/scenario-manifest.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/scenario-manifest.json new file mode 100644 index 0000000000000000000000000000000000000000..83eb11f577c7c9c7829aefad380ddf4e7bf1a6a0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/scenario-manifest.json @@ -0,0 +1,1299 @@ +{ + "scenario": { + "name": "pf_fm_experts", + "spec_file": "particleflow_spec.yaml", + "production_name": "cld", + "variants": { + "cld": { + "model_name": "pyg-cld-v1", + "production_name": null, + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "clic": { + "model_name": "pyg-clic-v1", + "production_name": "clic", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "idea": { + "model_name": "pyg-idea-pipeline-v1", + "production_name": "idea", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "colliderml": { + "model_name": "pyg-colliderml-v1", + "production_name": "colliderml", + "gpus": null, + "data_root": null, + "data_subdir": null, + "overrides": {} + }, + "cld_ecal_1p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_1p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_2p5mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_2p5mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_3mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_3mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_4mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_4mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_ecal_5p1mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "ecal_5p1mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_z_tautau_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_10mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_10mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_12mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_12mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_15mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_15mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_20mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_20mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_25mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_25mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + }, + "cld_hcal_30mm": { + "model_name": "pyg-cld-v1", + "production_name": "cld", + "gpus": null, + "data_root": "cld_granularity", + "data_subdir": "hcal_30mm/tfds", + "overrides": { + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + } + } + } + }, + "seeds": [ + 12345 + ], + "training": { + "global_batch_size": 16, + "parameters": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true + } + }, + "common_overrides": { + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6 + }, + "allowed_variant_differences": [ + "model.feature_bag", + "data_dir", + "dataset", + "hyperparameters.lr", + "input_dim", + "train_dataset", + "valid_dataset", + "test_dataset", + "enabled_test_datasets" + ] + }, + "platform": { + "name": "flatiron_h100_single", + "gpus": 1, + "python_command": [ + "uv", + "run", + "python3" + ], + "data_dir": { + "cld": "/mnt/ceph/users/jpata/mlpf/cld/v1.2.5_key4hep_2025-05-29/tfds", + "clic": "/mnt/ceph/users/jpata/mlpf/clic/v1.2.5_key4hep_2025-05-29/tfds", + "idea": "/mnt/ceph/users/jpata/mlpf/idea/IDEA_o1_v03_fccconfig_a05a3a9/tfds", + "colliderml": "/mnt/ceph/users/ewulff/tensorflow_datasets/colliderml/", + "cld_granularity": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29" + }, + "experiments_dir": "/mnt/home/jpata/particleflow/experiments", + "environment": {}, + "runtime_overrides": { + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false + }, + "slurm": { + "partition": "gpu", + "constraint": "h100", + "account": null, + "qos": null, + "reservation": null, + "time": "48:00:00", + "nodes": 1, + "tasks_per_node": 1, + "cpus_per_task": 64, + "cpus_per_gpu": 24, + "gpu_request": "gpus-per-task", + "gpu_type": null, + "memory": "128G", + "memory_per_gpu": null, + "no_requeue": false + } + }, + "job": { + "scenario_name": "pf_fm_experts", + "platform_name": "flatiron_h100_single", + "variant_name": "cld_hcal_30mm", + "model_name": "pyg-cld-v1", + "production_name": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_30mm/tfds", + "seed": 12345, + "global_batch_size": 16, + "per_gpu_batch_size": 16, + "gpu_batch_multiplier": 16, + "settings": { + "optimizer": "lamb", + "lr": 0.0005, + "lr_schedule": "onecycle", + "lr_schedule_config.onecycle.pct_start": 0.3, + "num_steps": 500000, + "tensorboard_step_freq": 1000, + "benchmark.enabled": true, + "benchmark.sample_interval_seconds": 1.0, + "benchmark.warmup_steps": 100, + "benchmark.gpu_timing_interval": 0, + "val_freq": 10000, + "checkpoint_freq": 2000, + "patience": 3, + "nvalid": 256, + "ntest": 256, + "sampler_mode": "interleaved-shards", + "pad_to_multiple_elements": 128, + "validation_particle_metrics": true, + "make_plots": true, + "model.type": "attention", + "model.heptv2": null, + "model.attention": { + "num_convs": 6, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_ff": 0.0 + }, + "model.input_encoding": "split", + "model.input_normalization": true, + "model.backbone.mode": "shared", + "model.backbone.num_convs": 6, + "dtype": "bfloat16", + "num_workers": 8, + "prefetch_factor": 2, + "model.attention.use_flash_attn_varlen": false, + "model.feature_bag": true, + "train_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "valid_dataset.cld.physical.samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ] + } + }, + "seed": 12345, + "gpu_batch_multiplier": 16 + } + }, + "resolved_config": { + "dataset": "cld", + "data_dir": "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_30mm/tfds", + "model": { + "type": "attention", + "input_encoding": "split", + "learned_representation_mode": "last", + "pt_mode": "direct-elemtype-split", + "eta_mode": "linear", + "sin_phi_mode": "linear", + "cos_phi_mode": "linear", + "energy_mode": "direct-elemtype-split", + "trainable": "all", + "task_queries": true, + "output_mode": "elementwise", + "binary_classification_focal_gamma": null, + "backbone": { + "mode": "shared", + "num_convs": 6, + "num_tracker_layers": null, + "num_calo_layers": null, + "num_common_layers": null + }, + "hit_feature_engineering": { + "enabled": true, + "geometry": true, + "tracker_neighborhood": false, + "tracker_momentum": false, + "tracker_doublets": false, + "calorimeter_neighborhood": true + }, + "set_decoder": null, + "input_normalization": true, + "feature_bag": true, + "gnnlsh": null, + "attention": { + "num_convs": 6, + "dropout_ff": 0.0, + "activation": "elu", + "layernorm": true, + "num_heads": 16, + "head_dim": 16, + "attention_type": "flash", + "dropout_conv_reg_mha": 0.0, + "dropout_conv_reg_ff": 0.0, + "dropout_conv_id_mha": 0.0, + "dropout_conv_id_ff": 0.0, + "use_pre_layernorm": true, + "use_jagged_attention": false, + "use_flash_attn_varlen": false, + "export_onnx_fused": false, + "save_attention": false + }, + "litept": null, + "hept": null, + "heptv2": null + }, + "conv_type": "attention", + "batch_size": 1, + "hyperparameters": { + "batch_size": 1, + "lr": 0.0005 + }, + "backend": "pytorch", + "threads": 16, + "gpu_type": "l40", + "mem_per_gpu_mb": 40000, + "slurm_partition": "gpu", + "slurm_runtime": "168h", + "input_dim": 19, + "num_classes": 6, + "elemtypes_nonzero": [ + 1, + 2 + ], + "seed": 12345, + "num_steps": 500000, + "patience": 3, + "checkpoint_freq": 2000, + "val_freq": 10000, + "num_workers": 8, + "prefetch_factor": 2, + "max_open_readers": 64, + "gpu_batch_multiplier": 16, + "dtype": "bfloat16", + "lr": 0.0005, + "weight_decay": 0.01, + "optimizer": "lamb", + "lr_schedule": "onecycle", + "lr_schedule_config": { + "onecycle": { + "pct_start": 0.3 + } + }, + "task_loss_weights": { + "calibration_steps": 100, + "epsilon": 1e-08, + "min_weight": 0.01, + "max_weight": 1000.0 + }, + "lr_finder": { + "start_lr": 1e-07, + "end_lr": 0.1, + "num_steps": 1000, + "smoothing": 0.98, + "divergence_factor": 4.0 + }, + "pad_to_multiple_elements": 128, + "validation_diagnostics_batches": 0, + "validation_linear_probe": false, + "validation_linear_probe_max_events": 16, + "validation_particle_metrics": true, + "train": true, + "lr_find": false, + "test": true, + "compile": false, + "make_plots": true, + "sort_data": false, + "sampler_mode": "interleaved-shards", + "load": null, + "relaxed_load": false, + "load_weights_only": false, + "reset_task_loss_weights": false, + "sampler_from_scratch": false, + "comet": false, + "comet_offline": false, + "comet_name": "particleflow", + "tensorboard_step_freq": 1000, + "benchmark": { + "enabled": true, + "sample_interval_seconds": 1.0, + "warmup_steps": 100, + "gpu_timing_interval": 0 + }, + "raytune": { + "local_dir": null, + "sched": "asha", + "search_alg": null, + "default_metric": "val_loss", + "default_mode": "min", + "asha": { + "max_t": 200, + "reduction_factor": 4, + "brackets": 1, + "grace_period": 10 + }, + "hyperband": { + "max_t": 200, + "reduction_factor": 4 + }, + "hyperopt": { + "n_random_steps": 10 + }, + "nevergrad": { + "n_random_steps": 10 + } + }, + "train_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "valid_dataset": { + "cld": { + "physical": { + "batch_size": 1, + "weight": null, + "samples": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": null + } + } + } + } + }, + "test_dataset": { + "cld_edm_ttbar_pf": { + "version": "3.3.0", + "splits": [ + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "10" + ], + "batch_size": 1 + } + }, + "enabled_test_datasets": [ + "cld_edm_ttbar_pf" + ], + "ntrain": null, + "nvalid": 256, + "ntest": 256, + "data_config": null, + "gpus": 1 + }, + "command": [ + "uv", + "run", + "python3", + "-u", + "mlpf/pipeline.py", + "--spec-file", + "particleflow_spec.yaml", + "--model-name", + "pyg-cld-v1", + "--production-name", + "cld", + "--data-dir", + "/mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_30mm/tfds", + "--experiment-dir", + "/mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994", + "train", + "--gpus", + "1", + "--dtype", + "bfloat16", + "--make-plots", + "--optimizer", + "lamb", + "--lr", + "0.0005", + "--lr_schedule", + "onecycle", + "--lr_schedule_config.onecycle.pct_start", + "0.3", + "--num_steps", + "500000", + "--tensorboard_step_freq", + "1000", + "--benchmark.enabled", + "true", + "--benchmark.sample_interval_seconds", + "1.0", + "--benchmark.warmup_steps", + "100", + "--benchmark.gpu_timing_interval", + "0", + "--val_freq", + "10000", + "--checkpoint_freq", + "2000", + "--patience", + "3", + "--nvalid", + "256", + "--ntest", + "256", + "--sampler_mode", + "interleaved-shards", + "--pad_to_multiple_elements", + "128", + "--validation_particle_metrics", + "true", + "--model.type", + "attention", + "--model.heptv2", + "null", + "--model.attention", + "{\"num_convs\": 6, \"num_heads\": 16, \"head_dim\": 16, \"attention_type\": \"flash\", \"dropout_ff\": 0.0}", + "--model.input_encoding", + "split", + "--model.input_normalization", + "true", + "--model.backbone.mode", + "shared", + "--model.backbone.num_convs", + "6", + "--num_workers", + "8", + "--prefetch_factor", + "2", + "--model.attention.use_flash_attn_varlen", + "false", + "--model.feature_bag", + "true", + "--train_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--valid_dataset.cld.physical.samples", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--test_dataset", + "{\"cld_edm_ttbar_pf\": {\"version\": \"3.3.0\", \"splits\": [\"1\", \"2\", \"3\", \"4\", \"5\", \"6\", \"7\", \"8\", \"9\", \"10\"]}}", + "--seed", + "12345", + "--gpu_batch_multiplier", + "16" + ], + "git_revision": "c68799620d27ceec2dc167db69d47fa42d3e84f7" +} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/train-config.yaml b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/train-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb36491e44f870f38e7b5c3c46301b86b4b09def --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/train-config.yaml @@ -0,0 +1,202 @@ +backend: pytorch +batch_size: 1 +benchmark: + enabled: true + gpu_timing_interval: 0 + sample_interval_seconds: 1.0 + warmup_steps: 100 +checkpoint_freq: 2000 +comet: false +comet_name: particleflow +comet_offline: false +compile: false +conv_type: attention +data_config: null +data_dir: /mnt/ceph/users/jpata/mlpf/cld/granularity_scan_2025-05-29/hcal_30mm/tfds +dataset: cld +dtype: bfloat16 +elemtypes_nonzero: +- 1 +- 2 +enabled_test_datasets: +- cld_edm_ttbar_pf +gpu_batch_multiplier: 16 +gpu_type: l40 +gpus: 1 +hyperparameters: + batch_size: 1 + lr: 0.0005 +input_dim: 19 +load: null +load_weights_only: false +lr: 0.0005 +lr_find: false +lr_finder: + divergence_factor: 4.0 + end_lr: 0.1 + num_steps: 1000 + smoothing: 0.98 + start_lr: 1.0e-07 +lr_schedule: onecycle +lr_schedule_config: + onecycle: + pct_start: 0.3 +make_plots: true +max_open_readers: 64 +mem_per_gpu_mb: 40000 +model: + attention: + activation: elu + attention_type: flash + dropout_conv_id_ff: 0.0 + dropout_conv_id_mha: 0.0 + dropout_conv_reg_ff: 0.0 + dropout_conv_reg_mha: 0.0 + dropout_ff: 0.0 + export_onnx_fused: false + head_dim: 16 + layernorm: true + num_convs: 6 + num_heads: 16 + save_attention: false + use_flash_attn_varlen: false + use_jagged_attention: false + use_pre_layernorm: true + backbone: + mode: shared + num_calo_layers: null + num_common_layers: null + num_convs: 6 + num_tracker_layers: null + binary_classification_focal_gamma: null + cos_phi_mode: linear + energy_mode: direct-elemtype-split + eta_mode: linear + feature_bag: true + gnnlsh: null + hept: null + heptv2: null + hit_feature_engineering: + calorimeter_neighborhood: true + enabled: true + geometry: true + tracker_doublets: false + tracker_momentum: false + tracker_neighborhood: false + input_encoding: split + input_normalization: true + learned_representation_mode: last + litept: null + output_mode: elementwise + pt_mode: direct-elemtype-split + set_decoder: null + sin_phi_mode: linear + task_queries: true + trainable: all + type: attention +ntest: 256 +ntrain: null +num_classes: 6 +num_steps: 500000 +num_workers: 8 +nvalid: 256 +optimizer: lamb +pad_to_multiple_elements: 128 +patience: 3 +prefetch_factor: 2 +raytune: + asha: + brackets: 1 + grace_period: 10 + max_t: 200 + reduction_factor: 4 + default_metric: val_loss + default_mode: min + hyperband: + max_t: 200 + reduction_factor: 4 + hyperopt: + n_random_steps: 10 + local_dir: null + nevergrad: + n_random_steps: 10 + sched: asha + search_alg: null +relaxed_load: false +reset_task_loss_weights: false +sampler_from_scratch: false +sampler_mode: interleaved-shards +seed: 12345 +slurm_partition: gpu +slurm_runtime: 168h +sort_data: false +task_loss_weights: + calibration_steps: 100 + epsilon: 1.0e-08 + max_weight: 1000.0 + min_weight: 0.01 +tensorboard_step_freq: 1000 +test: true +test_dataset: + cld_edm_ttbar_pf: + batch_size: 1 + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 +threads: 16 +train: true +train_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +val_freq: 10000 +valid_dataset: + cld: + physical: + batch_size: 1 + samples: + cld_edm_ttbar_pf: + batch_size: null + splits: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + version: 3.3.0 + weight: null +validation_diagnostics_batches: 0 +validation_linear_probe: false +validation_linear_probe_max_events: 16 +validation_particle_metrics: true +weight_decay: 0.01 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/train.log b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/train.log new file mode 100644 index 0000000000000000000000000000000000000000..725be611e73828bd53bcfcf949b869d7d4cfa1c1 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/train.log @@ -0,0 +1,1353 @@ +rank0 [2026-11-14 06:07:51,674] INFO : Will use single-gpu: NVIDIA H100 PCIe +rank0 [2026-11-14 06:07:51,674] INFO : Calling run(rank=0, world_size=1, ...) +rank0 [2026-11-14 06:07:52,151] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 06:07:52,151] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 06:07:52,153] INFO : Instantiating model +rank0 [2026-11-14 06:07:52,175] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 06:07:52,175] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 06:07:52,177] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 06:07:52,177] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 06:07:52,177] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 06:07:52,177] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 06:07:52,177] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,179] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 06:07:52,179] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,182] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 06:07:52,182] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,185] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 06:07:52,185] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,188] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 06:07:52,188] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,191] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 06:07:52,191] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,193] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 06:07:52,193] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 06:07:52,194] INFO : Initializing output DNNs +rank0 [2026-11-14 06:07:52,194] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,197] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,204] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 06:07:52,204] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 06:07:52,204] INFO : jagged_attention enabled=True +rank0 [2026-11-14 06:07:52,204] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 06:07:52,204] INFO : task_queries enabled=True +rank0 [2026-11-14 06:07:52,205] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 06:07:52,205] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 06:07:52,205] INFO : classification_query parameters: 256 +rank0 [2026-11-14 06:07:52,205] INFO : regression_query parameters: 256 +rank0 [2026-11-14 06:07:52,205] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 06:07:52,205] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 06:07:52,205] INFO : output_mode=elementwise +rank0 [2026-11-14 06:07:52,205] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 06:07:52,205] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 06:07:52,205] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 06:07:52,205] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 06:07:52,205] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 06:07:52,205] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 06:07:52,205] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 06:07:52,205] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 06:07:52,206] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 06:07:52,206] INFO : MLPF __init__ done +rank0 [2026-11-14 06:07:52,207] INFO : Instantiated model +rank0 [2026-11-14 06:07:52,207] INFO : Moving model to device rank=0 +rank0 [2026-11-14 06:07:52,215] INFO : Moved model to device rank=0 +rank0 [2026-11-14 06:07:52,216] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 06:07:52,313] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 06:07:52,313] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 06:07:52,314] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 06:07:52,314] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 06:07:52,314] INFO : Total parameters: 3910930 +rank0 [2026-11-14 06:07:52,316] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 06:07:52,322] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994 +rank0 [2026-11-14 06:07:52,322] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994 +rank0 [2026-11-14 06:07:52,322] INFO : Getting dataloaders +rank0 [2026-11-14 06:07:52,322] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:07:54,864] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:54,908] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:54,939] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:54,969] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,037] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,208] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,251] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,361] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,405] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,546] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,547] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 06:07:55,547] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 06:07:55,547] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 06:07:55,547] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:07:55,555] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,562] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,569] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,576] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,583] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,591] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,598] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,605] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,612] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,619] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,619] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 06:07:55,619] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 06:07:55,678] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:07:55,902] INFO : Got dataloaders +rank0 [2026-11-14 06:07:55,902] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 06:07:55,902] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 06:07:55,903] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 06:07:55,910] INFO : Creating train_iterator +rank0 [2026-11-14 06:07:55,910] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:55,939] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:07:56,101] INFO : Created train_iterator +rank0 [2026-11-14 06:08:08,355] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.089501957893372, 'Classification': 0.6290628957748413, 'Regression_pt': 0.5488760888576507, 'Regression_eta': 0.8808271092176437, 'Regression_sin_phi': 0.6515125647187233, 'Regression_cos_phi': 0.6336632177233696, 'Regression_energy': 0.6911959636211396}, weights={'Classification_binary': 1.0, 'Classification': 11.269941329956055, 'Regression_pt': 12.916398048400879, 'Regression_eta': 8.048686981201172, 'Regression_sin_phi': 10.881604194641113, 'Regression_cos_phi': 11.188122749328613, 'Regression_energy': 10.25686264038086} +rank0 [2026-11-14 06:08:59,017] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 06:08:59,017] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:46,863] INFO : Step 1000/500000 rank0 | Train Loss: 12.4256 | Train Loss Unweighted: 5.0118 | Train Loss Components: Classification: 0.0690 | Classification_binary: 4.2982 | Regression_cos_phi: 0.0233 | Regression_energy: 0.2759 | Regression_eta: 0.0322 | Regression_pt: 0.2907 | Regression_sin_phi: 0.0226 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.7781 | Classification_binary: 4.2982 | Regression_cos_phi: 0.2605 | Regression_energy: 2.8295 | Regression_eta: 0.2588 | Regression_pt: 3.7550 | Regression_sin_phi: 0.2456 | LR: 2.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5133s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0025 | max_backward: 0.0395 | max_optimizer_step: 0.0655 | max_tensorboard_logging: 0.1274 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2568.0 | uss_MB: 1349.1 | child_rss_MB: 22506.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:10:00,236] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 06:10:00,236] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:00,380] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 06:11:00,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:36,691] INFO : Step 2000/500000 rank0 | Train Loss: 11.8518 | Train Loss Unweighted: 5.0482 | Train Loss Components: Classification: 0.0526 | Classification_binary: 4.3975 | Regression_cos_phi: 0.0162 | Regression_energy: 0.2653 | Regression_eta: 0.0192 | Regression_pt: 0.2794 | Regression_sin_phi: 0.0180 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.5927 | Classification_binary: 4.3975 | Regression_cos_phi: 0.1813 | Regression_energy: 2.7211 | Regression_eta: 0.1545 | Regression_pt: 3.6084 | Regression_sin_phi: 0.1964 | LR: 2.02e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5335s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.1491 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2569.3 | uss_MB: 1350.4 | child_rss_MB: 22507.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:11:36,714] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 06:12:03,630] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 06:12:03,630] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:04,365] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 06:13:04,365] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:25,316] INFO : Step 3000/500000 rank0 | Train Loss: 9.9832 | Train Loss Unweighted: 4.6862 | Train Loss Components: Classification: 0.0416 | Classification_binary: 4.1766 | Regression_cos_phi: 0.0136 | Regression_energy: 0.2078 | Regression_eta: 0.0219 | Regression_pt: 0.2126 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4691 | Classification_binary: 4.1766 | Regression_cos_phi: 0.1520 | Regression_energy: 2.1312 | Regression_eta: 0.1765 | Regression_pt: 2.7458 | Regression_sin_phi: 0.1320 | LR: 2.05e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5237s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0027 | max_backward: 0.0408 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1191 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2600.1 | uss_MB: 1381.5 | child_rss_MB: 22509.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:14:05,334] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 06:14:05,334] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:05,388] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 06:15:05,388] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:12,901] INFO : Step 4000/500000 rank0 | Train Loss: 10.2073 | Train Loss Unweighted: 4.6368 | Train Loss Components: Classification: 0.0447 | Classification_binary: 4.1010 | Regression_cos_phi: 0.0155 | Regression_energy: 0.2175 | Regression_eta: 0.0227 | Regression_pt: 0.2230 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.5037 | Classification_binary: 4.1010 | Regression_cos_phi: 0.1738 | Regression_energy: 2.2313 | Regression_eta: 0.1826 | Regression_pt: 2.8806 | Regression_sin_phi: 0.1342 | LR: 2.08e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4924s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0382 | max_loss: 0.0027 | max_backward: 0.0391 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1161 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2600.2 | uss_MB: 1381.7 | child_rss_MB: 22511.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:15:12,905] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 06:16:06,005] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 06:16:06,005] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:00,595] INFO : Step 5000/500000 rank0 | Train Loss: 9.3765 | Train Loss Unweighted: 4.4695 | Train Loss Components: Classification: 0.0391 | Classification_binary: 3.9994 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1960 | Regression_eta: 0.0157 | Regression_pt: 0.2021 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4404 | Classification_binary: 3.9994 | Regression_cos_phi: 0.1170 | Regression_energy: 2.0104 | Regression_eta: 0.1267 | Regression_pt: 2.6103 | Regression_sin_phi: 0.0723 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4919s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0660 | max_tensorboard_logging: 0.1166 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2571.1 | uss_MB: 1352.5 | child_rss_MB: 22510.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:17:06,781] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 06:17:06,781] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:06,981] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 06:18:06,981] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:48,310] INFO : Step 6000/500000 rank0 | Train Loss: 9.8320 | Train Loss Unweighted: 4.5884 | Train Loss Components: Classification: 0.0354 | Classification_binary: 4.0881 | Regression_cos_phi: 0.0122 | Regression_energy: 0.2147 | Regression_eta: 0.0087 | Regression_pt: 0.2168 | Regression_sin_phi: 0.0125 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3992 | Classification_binary: 4.0881 | Regression_cos_phi: 0.1367 | Regression_energy: 2.2018 | Regression_eta: 0.0699 | Regression_pt: 2.7999 | Regression_sin_phi: 0.1363 | LR: 2.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5066s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0026 | max_backward: 0.0397 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1312 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2571.6 | uss_MB: 1353.1 | child_rss_MB: 22511.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:18:48,312] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 06:19:07,976] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 06:19:07,976] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:08,417] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 06:20:08,417] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:36,429] INFO : Step 7000/500000 rank0 | Train Loss: 8.8648 | Train Loss Unweighted: 4.3825 | Train Loss Components: Classification: 0.0387 | Classification_binary: 3.9531 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1811 | Regression_eta: 0.0108 | Regression_pt: 0.1795 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4360 | Classification_binary: 3.9531 | Regression_cos_phi: 0.1089 | Regression_energy: 1.8571 | Regression_eta: 0.0868 | Regression_pt: 2.3179 | Regression_sin_phi: 0.1050 | LR: 2.26e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5142s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0026 | max_backward: 0.0387 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.1242 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2601.6 | uss_MB: 1383.1 | child_rss_MB: 22512.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:21:09,732] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 06:21:09,732] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:10,263] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 06:22:10,263] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:24,909] INFO : Step 8000/500000 rank0 | Train Loss: 9.2742 | Train Loss Unweighted: 4.4956 | Train Loss Components: Classification: 0.0377 | Classification_binary: 4.0361 | Regression_cos_phi: 0.0167 | Regression_energy: 0.1863 | Regression_eta: 0.0141 | Regression_pt: 0.1842 | Regression_sin_phi: 0.0205 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4253 | Classification_binary: 4.0361 | Regression_cos_phi: 0.1873 | Regression_energy: 1.9104 | Regression_eta: 0.1132 | Regression_pt: 2.3790 | Regression_sin_phi: 0.2230 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5197s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0761 | max_tensorboard_logging: 0.1252 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2601.7 | uss_MB: 1383.2 | child_rss_MB: 22510.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:22:24,911] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 06:22:24,982] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 06:23:11,328] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 06:23:11,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:11,929] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 06:24:11,929] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:13,205] INFO : Step 9000/500000 rank0 | Train Loss: 8.6153 | Train Loss Unweighted: 4.4547 | Train Loss Components: Classification: 0.0311 | Classification_binary: 4.0567 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1680 | Regression_eta: 0.0103 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3505 | Classification_binary: 4.0567 | Regression_cos_phi: 0.1112 | Regression_energy: 1.7232 | Regression_eta: 0.0826 | Regression_pt: 2.2034 | Regression_sin_phi: 0.0876 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.5107s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0384 | max_optimizer_step: 0.0762 | max_tensorboard_logging: 0.1241 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2570.9 | uss_MB: 1352.4 | child_rss_MB: 22511.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:25:13,380] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 06:25:13,380] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:01,987] INFO : Step 10000/500000 rank0 | Train Loss: 8.4528 | Train Loss Unweighted: 4.0286 | Train Loss Components: Classification: 0.0353 | Classification_binary: 3.6051 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1822 | Regression_eta: 0.0094 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3980 | Classification_binary: 3.6051 | Regression_cos_phi: 0.0969 | Regression_energy: 1.8685 | Regression_eta: 0.0756 | Regression_pt: 2.3092 | Regression_sin_phi: 0.0994 | LR: 2.52e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4899s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0365 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1165 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2571.4 | uss_MB: 1352.8 | child_rss_MB: 22513.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:26:01,989] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 06:26:02,068] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 06:26:02,068] INFO : Running validation on rank0 +rank0 [2026-11-14 06:26:02,068] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:26:02,068] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:26:02,260] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 06:26:02,471] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:26:02,550] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:26:02,603] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:26:02,658] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:26:02,704] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:26:02,759] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:26:02,813] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:26:02,861] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:26:02,938] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:26:03,015] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:26:03,072] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:26:03,128] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:26:03,176] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:26:03,230] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:26:03,286] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:26:03,337] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:26:03,340] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:26:03,343] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:26:03,344] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:26:03,348] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:26:03,348] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:26:03,352] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:26:03,352] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:26:03,354] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:26:03,356] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:26:03,359] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:26:03,360] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:26:03,364] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:26:03,366] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:26:03,367] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:26:03,370] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:26:04,927] INFO : VALIDATION | Step=10000/500000 | Train Loss=8.4528 | Valid Loss=9.4338 | Train Loss Unweighted=4.0286 | Valid Loss Unweighted=4.4200 | Stale=0 | ETA=889.4m +rank0 [2026-11-14 06:26:04,928] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:26:04,999] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:26:05,006] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:26:05,068] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:26:05,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:26:06,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:26:06,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:26:06,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:26:06,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:26:07,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:26:07,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:26:07,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:26:07,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:26:08,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:26:08,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:26:08,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:26:08,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:26:09,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:26:09,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:26:09,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:26:09,778] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:26:09,784] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:26:10,358] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:26:11,245] INFO : Plotted number of elements +rank0 [2026-11-14 06:26:12,575] INFO : Plotted jets +rank0 [2026-11-14 06:26:13,959] INFO : Plotted jet ratio +rank0 [2026-11-14 06:26:26,755] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 06:26:26,755] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:27:27,490] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 06:27:27,491] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:02,584] INFO : Step 11000/500000 rank0 | Train Loss: 10.1441 | Train Loss Unweighted: 4.5251 | Train Loss Components: Classification: 0.0393 | Classification_binary: 3.9750 | Regression_cos_phi: 0.0160 | Regression_energy: 0.2330 | Regression_eta: 0.0407 | Regression_pt: 0.2080 | Regression_sin_phi: 0.0131 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4434 | Classification_binary: 3.9750 | Regression_cos_phi: 0.1786 | Regression_energy: 2.3897 | Regression_eta: 0.3273 | Regression_pt: 2.6871 | Regression_sin_phi: 0.1431 | LR: 2.63e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5187s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0773 | max_tensorboard_logging: 0.1228 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2738.1 | uss_MB: 1527.5 | child_rss_MB: 22514.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:28:28,304] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 06:28:28,304] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:29:28,448] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 06:29:28,448] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:29:50,046] INFO : Step 12000/500000 rank0 | Train Loss: 7.9452 | Train Loss Unweighted: 4.0182 | Train Loss Components: Classification: 0.0377 | Classification_binary: 3.6420 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1567 | Regression_eta: 0.0094 | Regression_pt: 0.1560 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4250 | Classification_binary: 3.6420 | Regression_cos_phi: 0.1009 | Regression_energy: 1.6069 | Regression_eta: 0.0756 | Regression_pt: 2.0151 | Regression_sin_phi: 0.0796 | LR: 2.75e-05 | DataLoad Time: 0.0055s | Model Forward Time: 0.5059s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0029 | max_backward: 0.0418 | max_optimizer_step: 0.0692 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2738.1 | uss_MB: 1527.5 | child_rss_MB: 22515.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:29:50,048] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 06:29:50,119] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 06:30:29,133] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 06:30:29,133] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:31:29,809] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 06:31:29,809] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:31:38,244] INFO : Step 13000/500000 rank0 | Train Loss: 10.1891 | Train Loss Unweighted: 4.2582 | Train Loss Components: Classification: 0.0384 | Classification_binary: 3.6945 | Regression_cos_phi: 0.0077 | Regression_energy: 0.2480 | Regression_eta: 0.0061 | Regression_pt: 0.2533 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4325 | Classification_binary: 3.6945 | Regression_cos_phi: 0.0864 | Regression_energy: 2.5438 | Regression_eta: 0.0491 | Regression_pt: 3.2711 | Regression_sin_phi: 0.1116 | LR: 2.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5399s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1320 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2678.7 | uss_MB: 1468.1 | child_rss_MB: 22515.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:32:30,810] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 06:32:30,810] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:33:26,432] INFO : Step 14000/500000 rank0 | Train Loss: 9.4320 | Train Loss Unweighted: 4.3154 | Train Loss Components: Classification: 0.0387 | Classification_binary: 3.8294 | Regression_cos_phi: 0.0064 | Regression_energy: 0.2101 | Regression_eta: 0.0057 | Regression_pt: 0.2182 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4360 | Classification_binary: 3.8294 | Regression_cos_phi: 0.0719 | Regression_energy: 2.1546 | Regression_eta: 0.0455 | Regression_pt: 2.8186 | Regression_sin_phi: 0.0761 | LR: 3.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5209s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0027 | max_backward: 0.0388 | max_optimizer_step: 0.0680 | max_tensorboard_logging: 0.1257 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2679.0 | uss_MB: 1468.4 | child_rss_MB: 22514.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:33:26,434] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:33:26,502] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 06:33:31,863] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 06:33:31,864] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:32,630] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 06:34:32,630] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:15,107] INFO : Step 15000/500000 rank0 | Train Loss: 8.8514 | Train Loss Unweighted: 4.0988 | Train Loss Components: Classification: 0.0334 | Classification_binary: 3.6459 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1946 | Regression_eta: 0.0074 | Regression_pt: 0.1982 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3766 | Classification_binary: 3.6459 | Regression_cos_phi: 0.1124 | Regression_energy: 1.9956 | Regression_eta: 0.0594 | Regression_pt: 2.5605 | Regression_sin_phi: 0.1009 | LR: 3.17e-05 | DataLoad Time: 0.0088s | Model Forward Time: 0.5089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0028 | max_backward: 0.0412 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2681.1 | uss_MB: 1470.5 | child_rss_MB: 22515.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:35:33,728] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 06:35:33,729] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:33,968] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 06:36:33,969] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:02,849] INFO : Step 16000/500000 rank0 | Train Loss: 9.5650 | Train Loss Unweighted: 4.4242 | Train Loss Components: Classification: 0.0337 | Classification_binary: 3.9337 | Regression_cos_phi: 0.0119 | Regression_energy: 0.2033 | Regression_eta: 0.0125 | Regression_pt: 0.2160 | Regression_sin_phi: 0.0131 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3797 | Classification_binary: 3.9337 | Regression_cos_phi: 0.1331 | Regression_energy: 2.0853 | Regression_eta: 0.1006 | Regression_pt: 2.7901 | Regression_sin_phi: 0.1426 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5007s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0378 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1234 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2681.2 | uss_MB: 1470.7 | child_rss_MB: 22518.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:37:02,851] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:37:02,919] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 06:37:34,885] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 06:37:34,885] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:34,650] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 06:38:34,650] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:49,949] INFO : Step 17000/500000 rank0 | Train Loss: 7.9654 | Train Loss Unweighted: 4.0320 | Train Loss Components: Classification: 0.0361 | Classification_binary: 3.6524 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1453 | Regression_eta: 0.0197 | Regression_pt: 0.1532 | Regression_sin_phi: 0.0138 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4071 | Classification_binary: 3.6524 | Regression_cos_phi: 0.1282 | Regression_energy: 1.4905 | Regression_eta: 0.1588 | Regression_pt: 1.9787 | Regression_sin_phi: 0.1498 | LR: 3.51e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5110s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0342 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0660 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2678.2 | uss_MB: 1467.6 | child_rss_MB: 22517.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:39:35,106] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 06:39:35,107] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:35,742] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 06:40:35,743] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:37,894] INFO : Step 18000/500000 rank0 | Train Loss: 8.3705 | Train Loss Unweighted: 4.1491 | Train Loss Components: Classification: 0.0396 | Classification_binary: 3.7480 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1718 | Regression_eta: 0.0037 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4467 | Classification_binary: 3.7480 | Regression_cos_phi: 0.0685 | Regression_energy: 1.7619 | Regression_eta: 0.0301 | Regression_pt: 2.2768 | Regression_sin_phi: 0.0385 | LR: 3.69e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5128s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2679.0 | uss_MB: 1468.4 | child_rss_MB: 22515.2 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 06:40:37,896] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:40:37,964] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 06:41:36,632] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 06:41:36,632] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:25,821] INFO : Step 19000/500000 rank0 | Train Loss: 8.3097 | Train Loss Unweighted: 4.2182 | Train Loss Components: Classification: 0.0307 | Classification_binary: 3.8275 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1664 | Regression_eta: 0.0076 | Regression_pt: 0.1681 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3456 | Classification_binary: 3.8275 | Regression_cos_phi: 0.0900 | Regression_energy: 1.7067 | Regression_eta: 0.0615 | Regression_pt: 2.1713 | Regression_sin_phi: 0.1072 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5179s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0343 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1312 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2680.9 | uss_MB: 1470.3 | child_rss_MB: 22517.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:42:37,314] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 06:42:37,314] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:43:37,425] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 06:43:37,425] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:44:13,282] INFO : Step 20000/500000 rank0 | Train Loss: 9.9121 | Train Loss Unweighted: 4.4119 | Train Loss Components: Classification: 0.0342 | Classification_binary: 3.8874 | Regression_cos_phi: 0.0066 | Regression_energy: 0.2306 | Regression_eta: 0.0097 | Regression_pt: 0.2330 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3857 | Classification_binary: 3.8874 | Regression_cos_phi: 0.0738 | Regression_energy: 2.3652 | Regression_eta: 0.0777 | Regression_pt: 3.0094 | Regression_sin_phi: 0.1128 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5235s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1333 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2680.9 | uss_MB: 1470.4 | child_rss_MB: 22515.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 06:44:13,284] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:44:13,352] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:44:13,353] INFO : Running validation on rank0 +rank0 [2026-11-14 06:44:13,353] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:44:13,353] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:44:13,503] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 06:44:13,623] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:44:13,735] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:44:13,804] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:44:13,869] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:44:13,933] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:44:13,999] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:44:14,066] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:44:14,182] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:44:14,295] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:44:14,407] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:44:14,474] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:44:14,539] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:44:14,603] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:44:14,670] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:44:14,737] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:44:14,836] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:44:14,842] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:44:14,845] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:44:14,848] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:44:14,850] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:44:14,852] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:44:14,854] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:44:14,856] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:44:14,858] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:44:14,860] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:44:14,861] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:44:14,864] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:44:14,866] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:44:14,868] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:44:14,870] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:44:14,872] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:44:16,418] INFO : VALIDATION | Step=20000/500000 | Train Loss=9.9121 | Valid Loss=9.0020 | Train Loss Unweighted=4.4119 | Valid Loss Unweighted=4.2325 | Stale=0 | ETA=872.2m +rank0 [2026-11-14 06:44:16,418] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:44:16,495] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:44:16,502] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:16,563] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:44:17,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:44:17,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:44:17,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:44:17,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:44:18,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:44:18,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:44:18,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:44:18,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:44:19,121] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:44:19,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:44:19,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:44:19,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:44:20,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:44:20,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:44:20,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:44:20,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:44:20,984] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:44:20,990] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:21,563] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:21,916] INFO : Plotted number of elements +rank0 [2026-11-14 06:44:23,223] INFO : Plotted jets +rank0 [2026-11-14 06:44:24,202] INFO : Plotted jet ratio +rank0 [2026-11-14 06:44:49,737] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 06:44:49,738] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:45:50,689] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 06:45:50,690] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:46:13,365] INFO : Step 21000/500000 rank0 | Train Loss: 9.4714 | Train Loss Unweighted: 4.2234 | Train Loss Components: Classification: 0.0316 | Classification_binary: 3.7215 | Regression_cos_phi: 0.0127 | Regression_energy: 0.2228 | Regression_eta: 0.0096 | Regression_pt: 0.2156 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3563 | Classification_binary: 3.7215 | Regression_cos_phi: 0.1421 | Regression_energy: 2.2855 | Regression_eta: 0.0774 | Regression_pt: 2.7848 | Regression_sin_phi: 0.1038 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5610s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0410 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1387 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2708.0 | uss_MB: 1497.7 | child_rss_MB: 22520.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:46:51,898] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 06:46:51,898] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:51,983] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 06:47:51,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:48:01,188] INFO : Step 22000/500000 rank0 | Train Loss: 8.4509 | Train Loss Unweighted: 3.9942 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.5687 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1843 | Regression_eta: 0.0088 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3818 | Classification_binary: 3.5687 | Regression_cos_phi: 0.0758 | Regression_energy: 1.8903 | Regression_eta: 0.0712 | Regression_pt: 2.3931 | Regression_sin_phi: 0.0700 | LR: 4.50e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5141s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0381 | max_loss: 0.0027 | max_backward: 0.0393 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1318 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2708.9 | uss_MB: 1498.5 | child_rss_MB: 22518.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:48:01,189] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:48:01,257] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:48:53,283] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 06:48:53,283] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:49,528] INFO : Step 23000/500000 rank0 | Train Loss: 8.0573 | Train Loss Unweighted: 4.0325 | Train Loss Components: Classification: 0.0253 | Classification_binary: 3.6491 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1700 | Regression_eta: 0.0056 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2847 | Classification_binary: 3.6491 | Regression_cos_phi: 0.0566 | Regression_energy: 1.7437 | Regression_eta: 0.0451 | Regression_pt: 2.2045 | Regression_sin_phi: 0.0736 | LR: 4.73e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5209s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0025 | max_backward: 0.0391 | max_optimizer_step: 0.0679 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2711.9 | uss_MB: 1501.5 | child_rss_MB: 22520.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:49:54,033] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 06:49:54,034] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:50:54,606] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 06:50:54,606] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:51:37,789] INFO : Step 24000/500000 rank0 | Train Loss: 8.8659 | Train Loss Unweighted: 4.1369 | Train Loss Components: Classification: 0.0309 | Classification_binary: 3.6863 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1962 | Regression_eta: 0.0069 | Regression_pt: 0.1987 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3481 | Classification_binary: 3.6863 | Regression_cos_phi: 0.0912 | Regression_energy: 2.0128 | Regression_eta: 0.0558 | Regression_pt: 2.5660 | Regression_sin_phi: 0.1058 | LR: 4.97e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5182s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0027 | max_backward: 0.0411 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1301 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2712.0 | uss_MB: 1501.6 | child_rss_MB: 22516.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:51:37,791] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:51:37,861] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:51:55,769] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 06:51:55,769] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:52:55,776] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 06:52:55,777] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:53:25,832] INFO : Step 25000/500000 rank0 | Train Loss: 8.0612 | Train Loss Unweighted: 4.0299 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.6457 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1678 | Regression_eta: 0.0069 | Regression_pt: 0.1700 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3184 | Classification_binary: 3.6457 | Regression_cos_phi: 0.0605 | Regression_energy: 1.7213 | Regression_eta: 0.0553 | Regression_pt: 2.1962 | Regression_sin_phi: 0.0638 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5114s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0029 | max_backward: 0.0393 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2708.1 | uss_MB: 1497.7 | child_rss_MB: 22518.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:53:56,928] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 06:53:56,928] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:54:57,134] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 06:54:57,134] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:55:13,461] INFO : Step 26000/500000 rank0 | Train Loss: 7.9863 | Train Loss Unweighted: 4.0461 | Train Loss Components: Classification: 0.0293 | Classification_binary: 3.6691 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1562 | Regression_eta: 0.0096 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3298 | Classification_binary: 3.6691 | Regression_cos_phi: 0.1276 | Regression_energy: 1.6025 | Regression_eta: 0.0772 | Regression_pt: 2.0601 | Regression_sin_phi: 0.1201 | LR: 5.47e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5296s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0359 | max_loss: 0.0028 | max_backward: 0.0390 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1342 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2709.4 | uss_MB: 1499.1 | child_rss_MB: 22518.8 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 06:55:13,463] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:55:13,529] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:55:57,855] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 06:55:57,855] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:56:58,477] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 06:56:58,477] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:57:01,495] INFO : Step 27000/500000 rank0 | Train Loss: 8.7978 | Train Loss Unweighted: 4.1366 | Train Loss Components: Classification: 0.0346 | Classification_binary: 3.6909 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1894 | Regression_eta: 0.0114 | Regression_pt: 0.1925 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3894 | Classification_binary: 3.6909 | Regression_cos_phi: 0.1083 | Regression_energy: 1.9425 | Regression_eta: 0.0921 | Regression_pt: 2.4859 | Regression_sin_phi: 0.0888 | LR: 5.74e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5225s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0025 | max_backward: 0.0387 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1240 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2712.1 | uss_MB: 1501.7 | child_rss_MB: 22519.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:58:00,015] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 06:58:00,015] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:58:50,121] INFO : Step 28000/500000 rank0 | Train Loss: 9.4271 | Train Loss Unweighted: 4.1574 | Train Loss Components: Classification: 0.0305 | Classification_binary: 3.6527 | Regression_cos_phi: 0.0108 | Regression_energy: 0.2160 | Regression_eta: 0.0146 | Regression_pt: 0.2180 | Regression_sin_phi: 0.0147 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3439 | Classification_binary: 3.6527 | Regression_cos_phi: 0.1206 | Regression_energy: 2.2158 | Regression_eta: 0.1177 | Regression_pt: 2.8160 | Regression_sin_phi: 0.1605 | LR: 6.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5197s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0378 | max_loss: 0.0027 | max_backward: 0.0395 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1358 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2712.6 | uss_MB: 1502.2 | child_rss_MB: 22518.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:58:50,123] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:58:50,190] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:59:00,999] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 06:59:00,999] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:00:01,434] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 07:00:01,434] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:00:38,413] INFO : Step 29000/500000 rank0 | Train Loss: 7.9264 | Train Loss Unweighted: 4.0552 | Train Loss Components: Classification: 0.0274 | Classification_binary: 3.6834 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1554 | Regression_eta: 0.0119 | Regression_pt: 0.1544 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3087 | Classification_binary: 3.6834 | Regression_cos_phi: 0.1166 | Regression_energy: 1.5940 | Regression_eta: 0.0962 | Regression_pt: 1.9946 | Regression_sin_phi: 0.1328 | LR: 6.29e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5257s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0399 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1351 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2712.6 | uss_MB: 1502.3 | child_rss_MB: 22517.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:01:02,631] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 07:01:02,631] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:02:03,209] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 07:02:03,209] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:02:26,519] INFO : Step 30000/500000 rank0 | Train Loss: 8.3337 | Train Loss Unweighted: 3.8369 | Train Loss Components: Classification: 0.0311 | Classification_binary: 3.4066 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1874 | Regression_eta: 0.0088 | Regression_pt: 0.1832 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3504 | Classification_binary: 3.4066 | Regression_cos_phi: 0.0856 | Regression_energy: 1.9223 | Regression_eta: 0.0707 | Regression_pt: 2.3660 | Regression_sin_phi: 0.1320 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5168s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0026 | max_backward: 0.0388 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1319 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2710.2 | uss_MB: 1499.8 | child_rss_MB: 22517.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:02:26,522] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 07:02:26,590] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 07:02:26,590] INFO : Running validation on rank0 +rank0 [2026-11-14 07:02:26,590] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:02:26,590] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:02:26,739] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 07:02:26,861] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:02:26,974] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:02:27,043] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:02:27,112] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:02:27,178] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:02:27,245] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:02:27,313] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:02:27,413] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:02:27,528] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:02:27,646] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:02:27,714] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:02:27,782] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:02:27,848] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:02:27,914] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:02:27,980] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:02:28,085] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:02:28,092] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:02:28,093] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:02:28,097] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:02:28,099] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:02:28,099] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:02:28,101] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:02:28,103] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:02:28,105] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:02:28,107] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:02:28,109] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:02:28,111] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:02:28,113] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:02:28,117] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:02:28,119] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:02:28,121] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:02:29,680] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.3337 | Valid Loss=8.7145 | Train Loss Unweighted=3.8369 | Valid Loss Unweighted=4.1070 | Stale=0 | ETA=854.8m +rank0 [2026-11-14 07:02:29,681] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:02:29,755] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:02:29,770] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:02:29,868] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:02:30,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:02:30,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:02:30,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:02:31,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:02:31,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:02:31,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:02:31,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:02:32,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:02:32,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:02:32,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:02:32,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:02:33,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:02:33,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:02:33,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:02:33,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:02:34,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:02:34,165] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:02:34,177] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:02:34,741] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:02:35,092] INFO : Plotted number of elements +rank0 [2026-11-14 07:02:36,396] INFO : Plotted jets +rank0 [2026-11-14 07:02:37,367] INFO : Plotted jet ratio +rank0 [2026-11-14 07:03:15,345] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 07:03:15,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:15,833] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 07:04:15,833] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:25,957] INFO : Step 31000/500000 rank0 | Train Loss: 7.8235 | Train Loss Unweighted: 3.9916 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.6230 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1548 | Regression_eta: 0.0126 | Regression_pt: 0.1510 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3352 | Classification_binary: 3.6230 | Regression_cos_phi: 0.1047 | Regression_energy: 1.5875 | Regression_eta: 0.1018 | Regression_pt: 1.9505 | Regression_sin_phi: 0.1208 | LR: 6.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5150s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0383 | max_loss: 0.0028 | max_backward: 0.0388 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1309 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2849.1 | uss_MB: 1638.8 | child_rss_MB: 22517.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:05:16,665] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 07:05:16,665] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:13,887] INFO : Step 32000/500000 rank0 | Train Loss: 8.4857 | Train Loss Unweighted: 3.6539 | Train Loss Components: Classification: 0.0353 | Classification_binary: 3.1917 | Regression_cos_phi: 0.0150 | Regression_energy: 0.1953 | Regression_eta: 0.0107 | Regression_pt: 0.1957 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3977 | Classification_binary: 3.1917 | Regression_cos_phi: 0.1674 | Regression_energy: 2.0032 | Regression_eta: 0.0862 | Regression_pt: 2.5281 | Regression_sin_phi: 0.1114 | LR: 7.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5181s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0381 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1272 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2861.9 | uss_MB: 1651.7 | child_rss_MB: 22517.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:06:13,889] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 07:06:13,956] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 07:06:17,618] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 07:06:17,618] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:07:18,451] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 07:07:18,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:08:01,686] INFO : Step 33000/500000 rank0 | Train Loss: 8.1361 | Train Loss Unweighted: 3.8920 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.4776 | Regression_cos_phi: 0.0168 | Regression_energy: 0.1781 | Regression_eta: 0.0254 | Regression_pt: 0.1547 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3365 | Classification_binary: 3.4776 | Regression_cos_phi: 0.1875 | Regression_energy: 1.8272 | Regression_eta: 0.2047 | Regression_pt: 1.9986 | Regression_sin_phi: 0.1041 | LR: 7.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5291s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0340 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0666 | max_tensorboard_logging: 0.1414 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2759.5 | uss_MB: 1559.7 | child_rss_MB: 22518.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:08:18,433] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 07:08:18,433] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:09:17,980] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 07:09:17,981] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:09:47,961] INFO : Step 34000/500000 rank0 | Train Loss: 8.3863 | Train Loss Unweighted: 3.8841 | Train Loss Components: Classification: 0.0371 | Classification_binary: 3.4540 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1832 | Regression_eta: 0.0077 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4185 | Classification_binary: 3.4540 | Regression_cos_phi: 0.1191 | Regression_energy: 1.8788 | Regression_eta: 0.0617 | Regression_pt: 2.3523 | Regression_sin_phi: 0.1018 | LR: 7.83e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5585s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0036 | max_backward: 0.0391 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.1425 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2763.1 | uss_MB: 1582.5 | child_rss_MB: 22519.8 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:09:47,965] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 07:09:48,037] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 07:10:17,704] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 07:10:17,705] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:11:17,612] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 07:11:17,612] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:11:34,530] INFO : Step 35000/500000 rank0 | Train Loss: 7.8065 | Train Loss Unweighted: 3.7042 | Train Loss Components: Classification: 0.0312 | Classification_binary: 3.3134 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1650 | Regression_eta: 0.0072 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3511 | Classification_binary: 3.3134 | Regression_cos_phi: 0.0955 | Regression_energy: 1.6921 | Regression_eta: 0.0581 | Regression_pt: 2.2195 | Regression_sin_phi: 0.0767 | LR: 8.16e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5251s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.1297 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2765.8 | uss_MB: 1598.0 | child_rss_MB: 22517.4 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:12:17,297] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 07:12:17,298] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:13:16,826] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 07:13:16,826] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:13:20,633] INFO : Step 36000/500000 rank0 | Train Loss: 7.8293 | Train Loss Unweighted: 3.8267 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.4447 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1666 | Regression_eta: 0.0062 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2868 | Classification_binary: 3.4447 | Regression_cos_phi: 0.1150 | Regression_energy: 1.7087 | Regression_eta: 0.0497 | Regression_pt: 2.1343 | Regression_sin_phi: 0.0901 | LR: 8.50e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5054s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0338 | max_loss: 0.0026 | max_backward: 0.0379 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1252 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2765.8 | uss_MB: 1598.0 | child_rss_MB: 22519.0 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:13:20,636] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 07:13:20,711] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 07:14:16,757] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 07:14:16,757] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:15:07,179] INFO : Step 37000/500000 rank0 | Train Loss: 8.7545 | Train Loss Unweighted: 3.7327 | Train Loss Components: Classification: 0.0374 | Classification_binary: 3.2523 | Regression_cos_phi: 0.0095 | Regression_energy: 0.2065 | Regression_eta: 0.0127 | Regression_pt: 0.2072 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4213 | Classification_binary: 3.2523 | Regression_cos_phi: 0.1059 | Regression_energy: 2.1184 | Regression_eta: 0.1026 | Regression_pt: 2.6759 | Regression_sin_phi: 0.0782 | LR: 8.85e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5116s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0355 | max_loss: 0.0027 | max_backward: 0.0381 | max_optimizer_step: 0.0657 | max_tensorboard_logging: 0.1323 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2765.8 | uss_MB: 1598.0 | child_rss_MB: 22519.1 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:15:16,900] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 07:15:16,900] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:16,867] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 07:16:16,867] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:53,866] INFO : Step 38000/500000 rank0 | Train Loss: 8.5823 | Train Loss Unweighted: 3.6180 | Train Loss Components: Classification: 0.0344 | Classification_binary: 3.1446 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2031 | Regression_eta: 0.0084 | Regression_pt: 0.2069 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3871 | Classification_binary: 3.1446 | Regression_cos_phi: 0.1070 | Regression_energy: 2.0830 | Regression_eta: 0.0678 | Regression_pt: 2.6723 | Regression_sin_phi: 0.1204 | LR: 9.21e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4972s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0384 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2762.9 | uss_MB: 1595.1 | child_rss_MB: 22518.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:16:53,869] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 07:16:53,940] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 07:17:16,697] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 07:17:16,697] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:18:16,450] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 07:18:16,450] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:18:40,447] INFO : Step 39000/500000 rank0 | Train Loss: 7.4667 | Train Loss Unweighted: 3.7185 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.3604 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1483 | Regression_eta: 0.0073 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3314 | Classification_binary: 3.3604 | Regression_cos_phi: 0.0995 | Regression_energy: 1.5210 | Regression_eta: 0.0589 | Regression_pt: 1.9619 | Regression_sin_phi: 0.1336 | LR: 9.57e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5126s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0027 | max_backward: 0.0385 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1163 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2765.5 | uss_MB: 1597.7 | child_rss_MB: 22519.0 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:19:16,549] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 07:19:16,549] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:20:16,496] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 07:20:16,496] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:20:27,314] INFO : Step 40000/500000 rank0 | Train Loss: 7.9207 | Train Loss Unweighted: 3.6823 | Train Loss Components: Classification: 0.0231 | Classification_binary: 3.2785 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1754 | Regression_eta: 0.0071 | Regression_pt: 0.1798 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2603 | Classification_binary: 3.2785 | Regression_cos_phi: 0.1103 | Regression_energy: 1.7985 | Regression_eta: 0.0574 | Regression_pt: 2.3229 | Regression_sin_phi: 0.0929 | LR: 9.94e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5025s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0368 | max_loss: 0.0029 | max_backward: 0.0394 | max_optimizer_step: 0.0704 | max_tensorboard_logging: 0.1212 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2766.2 | uss_MB: 1609.2 | child_rss_MB: 22518.8 | allocated_MB: 126.1 | reserved_MB: 652.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:20:27,316] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 07:20:27,387] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 07:20:27,388] INFO : Running validation on rank0 +rank0 [2026-11-14 07:20:27,388] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:20:27,388] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:20:27,538] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 07:20:27,655] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:20:27,764] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:20:27,829] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:20:27,892] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:20:27,954] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:20:28,018] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:20:28,085] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:20:28,179] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:20:28,290] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:20:28,401] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:20:28,468] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:20:28,532] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:20:28,595] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:20:28,659] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:20:28,725] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:20:28,829] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:20:28,836] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:20:28,837] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:20:28,839] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:20:28,842] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:20:28,844] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:20:28,846] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:20:28,848] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:20:28,850] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:20:28,851] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:20:28,854] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:20:28,856] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:20:28,858] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:20:28,860] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:20:28,862] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:20:28,864] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:20:30,359] INFO : VALIDATION | Step=40000/500000 | Train Loss=7.9207 | Valid Loss=8.2393 | Train Loss Unweighted=3.6823 | Valid Loss Unweighted=3.8649 | Stale=0 | ETA=834.6m +rank0 [2026-11-14 07:20:30,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:20:30,429] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:20:30,441] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:20:30,514] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:20:31,028] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:20:31,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:20:31,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:20:31,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:20:32,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:20:32,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:20:32,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:20:32,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:20:33,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:20:33,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:20:33,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:20:33,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:20:34,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:20:34,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:20:34,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:20:34,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:20:34,903] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:20:34,910] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:20:35,480] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:20:35,842] INFO : Plotted number of elements +rank0 [2026-11-14 07:20:37,167] INFO : Plotted jets +rank0 [2026-11-14 07:20:38,137] INFO : Plotted jet ratio +rank0 [2026-11-14 07:21:28,623] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 07:21:28,623] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:22:26,785] INFO : Step 41000/500000 rank0 | Train Loss: 7.8426 | Train Loss Unweighted: 3.8514 | Train Loss Components: Classification: 0.0296 | Classification_binary: 3.4686 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1642 | Regression_eta: 0.0101 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3339 | Classification_binary: 3.4686 | Regression_cos_phi: 0.1122 | Regression_energy: 1.6845 | Regression_eta: 0.0816 | Regression_pt: 2.0626 | Regression_sin_phi: 0.0991 | LR: 1.03e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5447s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0027 | max_backward: 0.0387 | max_optimizer_step: 0.0696 | max_tensorboard_logging: 0.1330 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2796.0 | uss_MB: 1644.2 | child_rss_MB: 22520.5 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:22:29,592] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 07:22:29,592] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:23:29,545] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 07:23:29,546] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:24:13,896] INFO : Step 42000/500000 rank0 | Train Loss: 8.6270 | Train Loss Unweighted: 4.0244 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.5860 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1926 | Regression_eta: 0.0061 | Regression_pt: 0.1934 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3555 | Classification_binary: 3.5860 | Regression_cos_phi: 0.0719 | Regression_energy: 1.9758 | Regression_eta: 0.0493 | Regression_pt: 2.4982 | Regression_sin_phi: 0.0903 | LR: 1.07e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5218s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1301 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.2 | uss_MB: 1678.4 | child_rss_MB: 22519.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:24:13,898] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 07:24:13,971] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 07:24:29,948] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 07:24:29,948] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:25:30,212] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 07:25:30,213] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:26:01,359] INFO : Step 43000/500000 rank0 | Train Loss: 7.3317 | Train Loss Unweighted: 3.6906 | Train Loss Components: Classification: 0.0302 | Classification_binary: 3.3432 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1489 | Regression_eta: 0.0047 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3398 | Classification_binary: 3.3432 | Regression_cos_phi: 0.0768 | Regression_energy: 1.5273 | Regression_eta: 0.0377 | Regression_pt: 1.9121 | Regression_sin_phi: 0.0948 | LR: 1.11e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5203s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0028 | max_backward: 0.0402 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.1313 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2839.3 | uss_MB: 1687.5 | child_rss_MB: 22519.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:26:30,578] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 07:26:30,578] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:27:30,623] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 07:27:30,623] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:27:48,653] INFO : Step 44000/500000 rank0 | Train Loss: 8.3316 | Train Loss Unweighted: 3.7067 | Train Loss Components: Classification: 0.0331 | Classification_binary: 3.2637 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1886 | Regression_eta: 0.0110 | Regression_pt: 0.1868 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3733 | Classification_binary: 3.2637 | Regression_cos_phi: 0.1270 | Regression_energy: 1.9341 | Regression_eta: 0.0889 | Regression_pt: 2.4131 | Regression_sin_phi: 0.1315 | LR: 1.15e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5247s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0403 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1304 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2857.4 | uss_MB: 1705.6 | child_rss_MB: 22518.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:27:48,655] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 07:27:48,724] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 07:28:31,433] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 07:28:31,433] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:31,518] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 07:29:31,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:36,183] INFO : Step 45000/500000 rank0 | Train Loss: 7.8993 | Train Loss Unweighted: 3.7366 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.3398 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1747 | Regression_eta: 0.0066 | Regression_pt: 0.1750 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3192 | Classification_binary: 3.3398 | Regression_cos_phi: 0.0625 | Regression_energy: 1.7915 | Regression_eta: 0.0530 | Regression_pt: 2.2610 | Regression_sin_phi: 0.0724 | LR: 1.19e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5054s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0030 | max_backward: 0.0400 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1296 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2873.5 | uss_MB: 1721.8 | child_rss_MB: 22519.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:30:31,551] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 07:30:31,552] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:31:22,651] INFO : Step 46000/500000 rank0 | Train Loss: 7.9525 | Train Loss Unweighted: 3.9603 | Train Loss Components: Classification: 0.0267 | Classification_binary: 3.5780 | Regression_cos_phi: 0.0128 | Regression_energy: 0.1621 | Regression_eta: 0.0088 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3013 | Classification_binary: 3.5780 | Regression_cos_phi: 0.1427 | Regression_energy: 1.6629 | Regression_eta: 0.0712 | Regression_pt: 2.0721 | Regression_sin_phi: 0.1244 | LR: 1.23e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4984s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0343 | max_loss: 0.0025 | max_backward: 0.0378 | max_optimizer_step: 0.0655 | max_tensorboard_logging: 0.1198 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2892.2 | uss_MB: 1740.4 | child_rss_MB: 22518.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:31:22,653] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:31:22,733] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 07:31:31,591] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 07:31:31,591] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:32:30,782] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 07:32:30,782] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:33:08,516] INFO : Step 47000/500000 rank0 | Train Loss: 7.3969 | Train Loss Unweighted: 3.6368 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.2787 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1558 | Regression_eta: 0.0052 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3362 | Classification_binary: 3.2787 | Regression_cos_phi: 0.0504 | Regression_energy: 1.5978 | Regression_eta: 0.0418 | Regression_pt: 2.0355 | Regression_sin_phi: 0.0566 | LR: 1.27e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.4923s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0641 | max_tensorboard_logging: 0.1269 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2934.4 | uss_MB: 1782.6 | child_rss_MB: 22518.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:33:30,708] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 07:33:30,708] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:34:30,596] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 07:34:30,596] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:34:55,519] INFO : Step 48000/500000 rank0 | Train Loss: 7.1036 | Train Loss Unweighted: 3.7263 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.4035 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1372 | Regression_eta: 0.0068 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2801 | Classification_binary: 3.4035 | Regression_cos_phi: 0.1058 | Regression_energy: 1.4077 | Regression_eta: 0.0545 | Regression_pt: 1.7767 | Regression_sin_phi: 0.0754 | LR: 1.31e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5050s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1273 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2936.2 | uss_MB: 1784.4 | child_rss_MB: 22519.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:34:55,520] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:34:55,587] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 07:35:31,096] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 07:35:31,096] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:36:30,381] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 07:36:30,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:36:42,056] INFO : Step 49000/500000 rank0 | Train Loss: 7.6632 | Train Loss Unweighted: 3.5841 | Train Loss Components: Classification: 0.0206 | Classification_binary: 3.1963 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1732 | Regression_eta: 0.0047 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2326 | Classification_binary: 3.1963 | Regression_cos_phi: 0.0605 | Regression_energy: 1.7767 | Regression_eta: 0.0378 | Regression_pt: 2.2781 | Regression_sin_phi: 0.0812 | LR: 1.36e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4944s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0398 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1292 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2936.2 | uss_MB: 1784.4 | child_rss_MB: 22519.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:37:30,568] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 07:37:30,568] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:38:29,311] INFO : Step 50000/500000 rank0 | Train Loss: 7.6179 | Train Loss Unweighted: 3.5243 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.1325 | Regression_cos_phi: 0.0123 | Regression_energy: 0.1667 | Regression_eta: 0.0078 | Regression_pt: 0.1635 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3312 | Classification_binary: 3.1325 | Regression_cos_phi: 0.1375 | Regression_energy: 1.7097 | Regression_eta: 0.0627 | Regression_pt: 2.1123 | Regression_sin_phi: 0.1321 | LR: 1.40e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5117s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0391 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1298 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2885.3 | uss_MB: 1733.5 | child_rss_MB: 22519.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:38:29,313] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:38:29,382] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 07:38:29,384] INFO : Running validation on rank0 +rank0 [2026-11-14 07:38:29,384] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:38:29,384] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:38:29,538] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 07:38:29,659] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:38:29,779] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:38:29,848] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:38:29,914] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:38:29,979] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:38:30,047] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:38:30,114] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:38:30,214] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:38:30,328] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:38:30,441] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:38:30,507] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:38:30,571] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:38:30,636] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:38:30,701] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:38:30,766] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:38:30,865] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:38:30,872] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:38:30,874] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:38:30,877] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:38:30,879] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:38:30,881] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:38:30,883] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:38:30,885] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:38:30,887] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:38:30,887] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:38:30,891] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:38:30,893] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:38:30,895] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:38:30,897] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:38:30,899] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:38:30,901] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:38:32,400] INFO : VALIDATION | Step=50000/500000 | Train Loss=7.6179 | Valid Loss=8.1213 | Train Loss Unweighted=3.5243 | Valid Loss Unweighted=3.7974 | Stale=0 | ETA=815.5m +rank0 [2026-11-14 07:38:32,400] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:38:32,475] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:38:32,483] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:38:32,563] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:38:33,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:38:33,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:38:33,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:38:33,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:38:34,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:38:34,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:38:34,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:38:34,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:38:35,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:38:35,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:38:35,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:38:35,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:38:36,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:38:36,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:38:36,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:38:36,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:38:36,904] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:38:36,910] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:38:37,483] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:38:37,830] INFO : Plotted number of elements +rank0 [2026-11-14 07:38:39,132] INFO : Plotted jets +rank0 [2026-11-14 07:38:40,099] INFO : Plotted jet ratio +rank0 [2026-11-14 07:38:42,446] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 07:38:42,446] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:39:43,352] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 07:39:43,352] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:40:28,844] INFO : Step 51000/500000 rank0 | Train Loss: 7.0042 | Train Loss Unweighted: 3.7067 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1224 | Regression_eta: 0.0096 | Regression_pt: 0.1375 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3059 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0913 | Regression_energy: 1.2556 | Regression_eta: 0.0775 | Regression_pt: 1.7756 | Regression_sin_phi: 0.1062 | LR: 1.44e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5187s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0345 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0671 | max_tensorboard_logging: 0.1304 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3018.6 | uss_MB: 1866.9 | child_rss_MB: 22519.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:40:44,011] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 07:40:44,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:41:44,335] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 07:41:44,335] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:42:16,810] INFO : Step 52000/500000 rank0 | Train Loss: 6.2117 | Train Loss Unweighted: 3.4412 | Train Loss Components: Classification: 0.0257 | Classification_binary: 3.1747 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1044 | Regression_eta: 0.0112 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2894 | Classification_binary: 3.1747 | Regression_cos_phi: 0.0745 | Regression_energy: 1.0712 | Regression_eta: 0.0901 | Regression_pt: 1.4096 | Regression_sin_phi: 0.1022 | LR: 1.49e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5225s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1298 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3018.6 | uss_MB: 1866.9 | child_rss_MB: 22528.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:42:16,812] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:42:16,882] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:42:45,616] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 07:42:45,616] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:43:46,734] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 07:43:46,734] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:44:05,750] INFO : Step 53000/500000 rank0 | Train Loss: 6.4144 | Train Loss Unweighted: 3.4659 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.1822 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1160 | Regression_eta: 0.0093 | Regression_pt: 0.1135 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2962 | Classification_binary: 3.1822 | Regression_cos_phi: 0.0862 | Regression_energy: 1.1901 | Regression_eta: 0.0748 | Regression_pt: 1.4658 | Regression_sin_phi: 0.1192 | LR: 1.53e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5169s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0385 | max_loss: 0.0027 | max_backward: 0.0391 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.1286 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3018.6 | uss_MB: 1866.9 | child_rss_MB: 22525.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:44:47,880] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 07:44:47,880] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:45:48,653] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 07:45:48,653] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:45:54,263] INFO : Step 54000/500000 rank0 | Train Loss: 7.0854 | Train Loss Unweighted: 3.5549 | Train Loss Components: Classification: 0.0273 | Classification_binary: 3.2149 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1479 | Regression_eta: 0.0110 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3081 | Classification_binary: 3.2149 | Regression_cos_phi: 0.0785 | Regression_energy: 1.5165 | Regression_eta: 0.0889 | Regression_pt: 1.7917 | Regression_sin_phi: 0.0867 | LR: 1.58e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5220s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1297 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2910.2 | uss_MB: 1758.5 | child_rss_MB: 22524.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:45:54,265] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:45:54,334] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:46:49,659] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 07:46:49,659] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:47:42,525] INFO : Step 55000/500000 rank0 | Train Loss: 6.0413 | Train Loss Unweighted: 3.5286 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.2859 | Regression_cos_phi: 0.0062 | Regression_energy: 0.0942 | Regression_eta: 0.0104 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3393 | Classification_binary: 3.2859 | Regression_cos_phi: 0.0694 | Regression_energy: 0.9659 | Regression_eta: 0.0838 | Regression_pt: 1.2040 | Regression_sin_phi: 0.0929 | LR: 1.62e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5126s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1276 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2914.0 | uss_MB: 1762.3 | child_rss_MB: 22525.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 32.3 +rank0 [2026-11-14 07:47:50,697] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 07:47:50,698] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:48:51,072] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 07:48:51,072] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:49:30,291] INFO : Step 56000/500000 rank0 | Train Loss: 7.0670 | Train Loss Unweighted: 3.4800 | Train Loss Components: Classification: 0.0251 | Classification_binary: 3.1364 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1440 | Regression_eta: 0.0089 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2824 | Classification_binary: 3.1364 | Regression_cos_phi: 0.1124 | Regression_energy: 1.4771 | Regression_eta: 0.0720 | Regression_pt: 1.8685 | Regression_sin_phi: 0.1182 | LR: 1.67e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5197s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0386 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1288 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2914.5 | uss_MB: 1762.8 | child_rss_MB: 22532.7 | allocated_MB: 125.7 | reserved_MB: 664.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:49:30,293] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:49:30,360] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:49:51,825] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 07:49:51,825] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:50:52,238] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 07:50:52,239] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:51:18,282] INFO : Step 57000/500000 rank0 | Train Loss: 6.5022 | Train Loss Unweighted: 3.3783 | Train Loss Components: Classification: 0.0261 | Classification_binary: 3.0772 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1239 | Regression_eta: 0.0124 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2939 | Classification_binary: 3.0772 | Regression_cos_phi: 0.0766 | Regression_energy: 1.2713 | Regression_eta: 0.1000 | Regression_pt: 1.5829 | Regression_sin_phi: 0.1004 | LR: 1.72e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5169s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1287 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2915.4 | uss_MB: 1763.7 | child_rss_MB: 22531.1 | allocated_MB: 125.7 | reserved_MB: 664.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:51:53,534] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 07:51:53,534] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:52:54,529] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 07:52:54,529] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:53:07,228] INFO : Step 58000/500000 rank0 | Train Loss: 6.4398 | Train Loss Unweighted: 3.4668 | Train Loss Components: Classification: 0.0275 | Classification_binary: 3.1796 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1170 | Regression_eta: 0.0126 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3095 | Classification_binary: 3.1796 | Regression_cos_phi: 0.0641 | Regression_energy: 1.1999 | Regression_eta: 0.1017 | Regression_pt: 1.4698 | Regression_sin_phi: 0.1154 | LR: 1.76e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5287s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0384 | max_optimizer_step: 0.0762 | max_tensorboard_logging: 0.1310 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2909.7 | uss_MB: 1758.0 | child_rss_MB: 22523.3 | allocated_MB: 125.7 | reserved_MB: 664.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:53:07,230] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:53:07,299] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:53:55,689] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 07:53:55,689] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:54:55,706] INFO : Step 59000/500000 rank0 | Train Loss: 6.0370 | Train Loss Unweighted: 3.3790 | Train Loss Components: Classification: 0.0280 | Classification_binary: 3.1229 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1027 | Regression_eta: 0.0088 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3152 | Classification_binary: 3.1229 | Regression_cos_phi: 0.1011 | Regression_energy: 1.0537 | Regression_eta: 0.0709 | Regression_pt: 1.2858 | Regression_sin_phi: 0.0874 | LR: 1.81e-04 | DataLoad Time: 0.0025s | Model Forward Time: 0.5111s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0029 | max_backward: 0.0412 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.1307 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2913.5 | uss_MB: 1761.8 | child_rss_MB: 22518.1 | allocated_MB: 125.7 | reserved_MB: 664.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:54:56,791] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 07:54:56,791] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:55:57,270] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 07:55:57,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:56:43,955] INFO : Step 60000/500000 rank0 | Train Loss: 6.8870 | Train Loss Unweighted: 3.6523 | Train Loss Components: Classification: 0.0259 | Classification_binary: 3.3404 | Regression_cos_phi: 0.0138 | Regression_energy: 0.1205 | Regression_eta: 0.0147 | Regression_pt: 0.1252 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2921 | Classification_binary: 3.3404 | Regression_cos_phi: 0.1549 | Regression_energy: 1.2359 | Regression_eta: 0.1180 | Regression_pt: 1.6176 | Regression_sin_phi: 0.1282 | LR: 1.86e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5317s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1465 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2913.5 | uss_MB: 1761.8 | child_rss_MB: 22521.8 | allocated_MB: 125.7 | reserved_MB: 664.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:56:43,957] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:56:44,024] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:56:44,025] INFO : Running validation on rank0 +rank0 [2026-11-14 07:56:44,025] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:56:44,025] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:56:44,174] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 07:56:44,294] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:56:44,407] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:56:44,475] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:56:44,540] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:56:44,604] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:56:44,670] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:56:44,735] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:56:44,834] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:56:44,947] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:56:45,061] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:56:45,130] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:56:45,195] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:56:45,260] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:56:45,327] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:56:45,394] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:56:45,502] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:56:45,509] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:56:45,511] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:56:45,512] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:56:45,516] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:56:45,518] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:56:45,520] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:56:45,522] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:56:45,524] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:56:45,526] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:56:45,528] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:56:45,530] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:56:45,532] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:56:45,534] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:56:45,536] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:56:45,538] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:56:47,007] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.8870 | Valid Loss=8.5005 | Train Loss Unweighted=3.6523 | Valid Loss Unweighted=3.8441 | Stale=1 | ETA=798.2m +rank0 [2026-11-14 07:56:47,007] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:56:47,084] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:56:47,092] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:56:47,155] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:56:47,687] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:56:47,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:56:48,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:56:48,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:56:48,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:56:48,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:56:49,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:56:49,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:56:49,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:56:49,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:56:50,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:56:50,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:56:50,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:56:50,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:56:51,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:56:51,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:56:51,522] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:56:51,527] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:56:52,106] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:56:52,462] INFO : Plotted number of elements +rank0 [2026-11-14 07:56:53,758] INFO : Plotted jets +rank0 [2026-11-14 07:56:54,726] INFO : Plotted jet ratio +rank0 [2026-11-14 07:57:09,490] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 07:57:09,491] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:58:09,932] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 07:58:09,933] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:58:43,054] INFO : Step 61000/500000 rank0 | Train Loss: 6.6271 | Train Loss Unweighted: 3.4380 | Train Loss Components: Classification: 0.0325 | Classification_binary: 3.1311 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1249 | Regression_eta: 0.0109 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3661 | Classification_binary: 3.1311 | Regression_cos_phi: 0.0933 | Regression_energy: 1.2812 | Regression_eta: 0.0878 | Regression_pt: 1.5887 | Regression_sin_phi: 0.0789 | LR: 1.91e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5399s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0027 | max_backward: 0.0386 | max_optimizer_step: 0.0767 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3041.4 | uss_MB: 1889.7 | child_rss_MB: 22527.2 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:59:10,840] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 07:59:10,841] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:11,195] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 08:00:11,195] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:31,013] INFO : Step 62000/500000 rank0 | Train Loss: 6.0527 | Train Loss Unweighted: 3.1848 | Train Loss Components: Classification: 0.0295 | Classification_binary: 2.9086 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1132 | Regression_eta: 0.0104 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3324 | Classification_binary: 2.9086 | Regression_cos_phi: 0.0691 | Regression_energy: 1.1608 | Regression_eta: 0.0834 | Regression_pt: 1.4301 | Regression_sin_phi: 0.0683 | LR: 1.95e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5114s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1293 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2934.8 | uss_MB: 1783.1 | child_rss_MB: 22528.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:00:31,015] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 08:00:31,082] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 08:01:12,037] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 08:01:12,037] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:12,711] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 08:02:12,711] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:19,232] INFO : Step 63000/500000 rank0 | Train Loss: 5.8775 | Train Loss Unweighted: 3.2137 | Train Loss Components: Classification: 0.0245 | Classification_binary: 2.9575 | Regression_cos_phi: 0.0047 | Regression_energy: 0.1084 | Regression_eta: 0.0091 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2762 | Classification_binary: 2.9575 | Regression_cos_phi: 0.0527 | Regression_energy: 1.1116 | Regression_eta: 0.0730 | Regression_pt: 1.3627 | Regression_sin_phi: 0.0438 | LR: 2.00e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5237s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1318 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2937.8 | uss_MB: 1786.1 | child_rss_MB: 22530.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:03:13,780] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 08:03:13,780] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:04:07,568] INFO : Step 64000/500000 rank0 | Train Loss: 5.7416 | Train Loss Unweighted: 3.2509 | Train Loss Components: Classification: 0.0246 | Classification_binary: 3.0107 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1002 | Regression_eta: 0.0079 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2773 | Classification_binary: 3.0107 | Regression_cos_phi: 0.0740 | Regression_energy: 1.0281 | Regression_eta: 0.0637 | Regression_pt: 1.2091 | Regression_sin_phi: 0.0787 | LR: 2.05e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5220s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.1290 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2937.9 | uss_MB: 1786.2 | child_rss_MB: 22527.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:04:07,569] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 08:04:07,636] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 08:04:14,932] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 08:04:14,932] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:15,545] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 08:05:15,545] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:56,439] INFO : Step 65000/500000 rank0 | Train Loss: 5.4065 | Train Loss Unweighted: 3.1965 | Train Loss Components: Classification: 0.0266 | Classification_binary: 2.9827 | Regression_cos_phi: 0.0062 | Regression_energy: 0.0806 | Regression_eta: 0.0103 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3001 | Classification_binary: 2.9827 | Regression_cos_phi: 0.0694 | Regression_energy: 0.8263 | Regression_eta: 0.0827 | Regression_pt: 1.0451 | Regression_sin_phi: 0.1002 | LR: 2.10e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5319s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0027 | max_backward: 0.0389 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1323 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2939.0 | uss_MB: 1787.3 | child_rss_MB: 22535.1 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 08:06:17,201] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 08:06:17,202] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:07:17,793] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 08:07:17,794] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:07:44,865] INFO : Step 66000/500000 rank0 | Train Loss: 5.6839 | Train Loss Unweighted: 3.0924 | Train Loss Components: Classification: 0.0283 | Classification_binary: 2.8420 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1040 | Regression_eta: 0.0097 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3195 | Classification_binary: 2.8420 | Regression_cos_phi: 0.0710 | Regression_energy: 1.0667 | Regression_eta: 0.0778 | Regression_pt: 1.2461 | Regression_sin_phi: 0.0608 | LR: 2.15e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5303s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.1332 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2934.2 | uss_MB: 1782.5 | child_rss_MB: 22536.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:07:44,867] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 08:07:44,937] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 08:08:19,032] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 08:08:19,032] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:09:19,740] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 08:09:19,740] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:09:33,355] INFO : Step 67000/500000 rank0 | Train Loss: 5.5839 | Train Loss Unweighted: 3.0728 | Train Loss Components: Classification: 0.0245 | Classification_binary: 2.8314 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0964 | Regression_eta: 0.0084 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2758 | Classification_binary: 2.8314 | Regression_cos_phi: 0.0644 | Regression_energy: 0.9889 | Regression_eta: 0.0676 | Regression_pt: 1.2584 | Regression_sin_phi: 0.0974 | LR: 2.20e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5248s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0027 | max_backward: 0.0398 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.1285 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2937.7 | uss_MB: 1786.0 | child_rss_MB: 22535.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 08:10:21,300] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 08:10:21,300] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:11:22,265] INFO : Step 68000/500000 rank0 | Train Loss: 6.0404 | Train Loss Unweighted: 3.2520 | Train Loss Components: Classification: 0.0266 | Classification_binary: 2.9847 | Regression_cos_phi: 0.0125 | Regression_energy: 0.1061 | Regression_eta: 0.0068 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2994 | Classification_binary: 2.9847 | Regression_cos_phi: 0.1403 | Regression_energy: 1.0880 | Regression_eta: 0.0544 | Regression_pt: 1.3842 | Regression_sin_phi: 0.0894 | LR: 2.25e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5222s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0687 | max_tensorboard_logging: 0.1299 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2937.9 | uss_MB: 1786.2 | child_rss_MB: 22535.4 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 7.9 +rank0 [2026-11-14 08:11:22,267] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 08:11:22,337] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 08:11:22,555] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 08:11:22,555] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:12:23,418] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 08:12:23,418] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:13:10,966] INFO : Step 69000/500000 rank0 | Train Loss: 6.3528 | Train Loss Unweighted: 3.1760 | Train Loss Components: Classification: 0.0248 | Classification_binary: 2.8714 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1274 | Regression_eta: 0.0089 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2796 | Classification_binary: 2.8714 | Regression_cos_phi: 0.0864 | Regression_energy: 1.3070 | Regression_eta: 0.0720 | Regression_pt: 1.6467 | Regression_sin_phi: 0.0897 | LR: 2.30e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5126s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0026 | max_backward: 0.0399 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1290 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2937.8 | uss_MB: 1786.1 | child_rss_MB: 22538.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 28.3 +rank0 [2026-11-14 08:13:24,569] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 08:13:24,569] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:14:25,895] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 08:14:25,895] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:14:59,995] INFO : Step 70000/500000 rank0 | Train Loss: 5.9756 | Train Loss Unweighted: 3.1725 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.9051 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1114 | Regression_eta: 0.0063 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2339 | Classification_binary: 2.9051 | Regression_cos_phi: 0.0703 | Regression_energy: 1.1427 | Regression_eta: 0.0506 | Regression_pt: 1.5112 | Regression_sin_phi: 0.0619 | LR: 2.35e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5135s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1275 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2934.2 | uss_MB: 1782.5 | child_rss_MB: 22540.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 08:14:59,997] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 08:15:00,070] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 08:15:00,071] INFO : Running validation on rank0 +rank0 [2026-11-14 08:15:00,071] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 08:15:00,071] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 08:15:00,220] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 08:15:00,344] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 08:15:00,459] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 08:15:00,529] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 08:15:00,596] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 08:15:00,662] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 08:15:00,729] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 08:15:00,796] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 08:15:00,899] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 08:15:01,014] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 08:15:01,130] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 08:15:01,196] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 08:15:01,261] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 08:15:01,325] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 08:15:01,391] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 08:15:01,455] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 08:15:01,567] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 08:15:01,574] INFO : plotted pt_target histogram +rank0 [2026-11-14 08:15:01,577] INFO : plotted pt_pred histogram +rank0 [2026-11-14 08:15:01,579] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 08:15:01,581] INFO : plotted eta_target histogram +rank0 [2026-11-14 08:15:01,583] INFO : plotted eta_pred histogram +rank0 [2026-11-14 08:15:01,585] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 08:15:01,587] INFO : plotted sphi_target histogram +rank0 [2026-11-14 08:15:01,589] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 08:15:01,591] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 08:15:01,592] INFO : plotted cphi_target histogram +rank0 [2026-11-14 08:15:01,595] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 08:15:01,597] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 08:15:01,599] INFO : plotted energy_target histogram +rank0 [2026-11-14 08:15:01,600] INFO : plotted energy_pred histogram +rank0 [2026-11-14 08:15:01,603] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 08:15:03,074] INFO : VALIDATION | Step=70000/500000 | Train Loss=5.9756 | Valid Loss=9.2647 | Train Loss Unweighted=3.1725 | Valid Loss Unweighted=3.9874 | Stale=2 | ETA=780.9m +rank0 [2026-11-14 08:15:03,074] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 08:15:03,151] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 08:15:03,160] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 08:15:03,224] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 08:15:03,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 08:15:04,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 08:15:04,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 08:15:04,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 08:15:04,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 08:15:05,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 08:15:05,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 08:15:05,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 08:15:05,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 08:15:06,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 08:15:06,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 08:15:06,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 08:15:06,850] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 08:15:07,099] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 08:15:07,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 08:15:07,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 08:15:07,660] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 08:15:07,666] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:15:08,247] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:15:08,595] INFO : Plotted number of elements +rank0 [2026-11-14 08:15:09,901] INFO : Plotted jets +rank0 [2026-11-14 08:15:10,872] INFO : Plotted jet ratio +rank0 [2026-11-14 08:15:38,364] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 08:15:38,364] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:16:38,972] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 08:16:38,973] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:16:59,607] INFO : Step 71000/500000 rank0 | Train Loss: 5.5822 | Train Loss Unweighted: 2.9797 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.7183 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1032 | Regression_eta: 0.0363 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2489 | Classification_binary: 2.7183 | Regression_cos_phi: 0.0957 | Regression_energy: 1.0588 | Regression_eta: 0.2924 | Regression_pt: 1.1121 | Regression_sin_phi: 0.0559 | LR: 2.40e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5374s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0780 | max_tensorboard_logging: 0.1305 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3070.7 | uss_MB: 1919.0 | child_rss_MB: 22541.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:17:39,854] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 08:17:39,855] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:18:40,475] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 08:18:40,476] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:18:47,792] INFO : Step 72000/500000 rank0 | Train Loss: 5.5757 | Train Loss Unweighted: 3.1282 | Train Loss Components: Classification: 0.0252 | Classification_binary: 2.8933 | Regression_cos_phi: 0.0057 | Regression_energy: 0.0969 | Regression_eta: 0.0066 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2835 | Classification_binary: 2.8933 | Regression_cos_phi: 0.0634 | Regression_energy: 0.9938 | Regression_eta: 0.0532 | Regression_pt: 1.2334 | Regression_sin_phi: 0.0552 | LR: 2.45e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5331s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0390 | max_loss: 0.0028 | max_backward: 0.0404 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1292 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3070.7 | uss_MB: 1919.0 | child_rss_MB: 22540.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:18:47,794] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 08:18:47,864] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 08:19:41,935] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 08:19:41,936] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:20:36,523] INFO : Step 73000/500000 rank0 | Train Loss: 5.3438 | Train Loss Unweighted: 2.9959 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.7711 | Regression_cos_phi: 0.0081 | Regression_energy: 0.0931 | Regression_eta: 0.0055 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2117 | Classification_binary: 2.7711 | Regression_cos_phi: 0.0905 | Regression_energy: 0.9549 | Regression_eta: 0.0439 | Regression_pt: 1.2106 | Regression_sin_phi: 0.0610 | LR: 2.50e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5180s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0395 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1289 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3071.0 | uss_MB: 1919.3 | child_rss_MB: 22546.1 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 08:20:42,965] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 08:20:42,965] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:21:43,635] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 08:21:43,635] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:22:24,887] INFO : Step 74000/500000 rank0 | Train Loss: 4.9254 | Train Loss Unweighted: 2.6455 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4258 | Regression_cos_phi: 0.0084 | Regression_energy: 0.0871 | Regression_eta: 0.0084 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2197 | Classification_binary: 2.4258 | Regression_cos_phi: 0.0935 | Regression_energy: 0.8939 | Regression_eta: 0.0677 | Regression_pt: 1.1230 | Regression_sin_phi: 0.1018 | LR: 2.55e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5270s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0031 | max_backward: 0.0400 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1416 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2961.2 | uss_MB: 1809.5 | child_rss_MB: 22547.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:22:24,890] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 08:22:24,962] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 08:22:44,804] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 08:22:44,804] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:23:45,473] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 08:23:45,473] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:24:13,737] INFO : Step 75000/500000 rank0 | Train Loss: 5.3437 | Train Loss Unweighted: 2.7424 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.4927 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1060 | Regression_eta: 0.0080 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2483 | Classification_binary: 2.4927 | Regression_cos_phi: 0.0593 | Regression_energy: 1.0872 | Regression_eta: 0.0643 | Regression_pt: 1.3489 | Regression_sin_phi: 0.0430 | LR: 2.60e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5150s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0659 | max_tensorboard_logging: 0.1297 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2965.1 | uss_MB: 1813.4 | child_rss_MB: 22548.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:24:47,084] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 08:24:47,084] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:25:47,632] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 08:25:47,632] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:26:02,045] INFO : Step 76000/500000 rank0 | Train Loss: 4.3524 | Train Loss Unweighted: 2.4120 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.2258 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0768 | Regression_eta: 0.0042 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2608 | Classification_binary: 2.2258 | Regression_cos_phi: 0.0478 | Regression_energy: 0.7879 | Regression_eta: 0.0338 | Regression_pt: 0.9518 | Regression_sin_phi: 0.0445 | LR: 2.65e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5191s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0692 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2965.2 | uss_MB: 1813.5 | child_rss_MB: 22548.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:26:02,047] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 08:26:02,115] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 08:26:48,636] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 08:26:48,636] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:27:49,437] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 08:27:49,437] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:27:50,597] INFO : Step 77000/500000 rank0 | Train Loss: 4.6305 | Train Loss Unweighted: 2.6613 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.4680 | Regression_cos_phi: 0.0093 | Regression_energy: 0.0743 | Regression_eta: 0.0139 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2629 | Classification_binary: 2.4680 | Regression_cos_phi: 0.1037 | Regression_energy: 0.7622 | Regression_eta: 0.1122 | Regression_pt: 0.8409 | Regression_sin_phi: 0.0808 | LR: 2.70e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.6132s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0027 | max_backward: 0.0386 | max_optimizer_step: 0.0660 | max_tensorboard_logging: 0.2219 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2966.1 | uss_MB: 1814.4 | child_rss_MB: 22548.5 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 7.9 +rank0 [2026-11-14 08:28:50,780] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 08:28:50,780] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:29:39,065] INFO : Step 78000/500000 rank0 | Train Loss: 5.0095 | Train Loss Unweighted: 2.8530 | Train Loss Components: Classification: 0.0255 | Classification_binary: 2.6452 | Regression_cos_phi: 0.0068 | Regression_energy: 0.0807 | Regression_eta: 0.0072 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2875 | Classification_binary: 2.6452 | Regression_cos_phi: 0.0765 | Regression_energy: 0.8272 | Regression_eta: 0.0583 | Regression_pt: 1.0322 | Regression_sin_phi: 0.0826 | LR: 2.75e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5968s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0393 | max_loss: 0.0027 | max_backward: 0.0389 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.2105 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2965.9 | uss_MB: 1814.2 | child_rss_MB: 22551.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 08:29:39,067] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 08:29:39,136] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 08:29:51,870] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 08:29:51,870] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:30:52,339] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 08:30:52,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:31:27,346] INFO : Step 79000/500000 rank0 | Train Loss: 4.8829 | Train Loss Unweighted: 2.5650 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.3422 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0902 | Regression_eta: 0.0069 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2573 | Classification_binary: 2.3422 | Regression_cos_phi: 0.0672 | Regression_energy: 0.9253 | Regression_eta: 0.0556 | Regression_pt: 1.1516 | Regression_sin_phi: 0.0838 | LR: 2.80e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5828s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1997 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2965.6 | uss_MB: 1813.9 | child_rss_MB: 22554.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:31:53,438] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 08:31:53,439] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:32:54,492] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 08:32:54,492] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:33:16,072] INFO : Step 80000/500000 rank0 | Train Loss: 4.3980 | Train Loss Unweighted: 2.5903 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4148 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0643 | Regression_eta: 0.0113 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2161 | Classification_binary: 2.4148 | Regression_cos_phi: 0.0768 | Regression_energy: 0.6599 | Regression_eta: 0.0908 | Regression_pt: 0.8631 | Regression_sin_phi: 0.0764 | LR: 2.85e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5712s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0387 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1805 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2965.7 | uss_MB: 1814.0 | child_rss_MB: 22553.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:33:16,074] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 08:33:16,141] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 08:33:16,142] INFO : Running validation on rank0 +rank0 [2026-11-14 08:33:16,142] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 08:33:16,142] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 08:33:16,299] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 08:33:16,421] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 08:33:16,538] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 08:33:16,607] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 08:33:16,671] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 08:33:16,735] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 08:33:16,800] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 08:33:16,865] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 08:33:16,968] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 08:33:17,084] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 08:33:17,196] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 08:33:17,263] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 08:33:17,328] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 08:33:17,391] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 08:33:17,455] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 08:33:17,520] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 08:33:17,622] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 08:33:17,629] INFO : plotted pt_target histogram +rank0 [2026-11-14 08:33:17,631] INFO : plotted pt_pred histogram +rank0 [2026-11-14 08:33:17,633] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 08:33:17,635] INFO : plotted eta_target histogram +rank0 [2026-11-14 08:33:17,637] INFO : plotted eta_pred histogram +rank0 [2026-11-14 08:33:17,639] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 08:33:17,641] INFO : plotted sphi_target histogram +rank0 [2026-11-14 08:33:17,643] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 08:33:17,645] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 08:33:17,647] INFO : plotted cphi_target histogram +rank0 [2026-11-14 08:33:17,648] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 08:33:17,651] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 08:33:17,653] INFO : plotted energy_target histogram +rank0 [2026-11-14 08:33:17,654] INFO : plotted energy_pred histogram +rank0 [2026-11-14 08:33:17,658] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 08:33:19,121] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.3980 | Valid Loss=10.1848 | Train Loss Unweighted=2.5903 | Valid Loss Unweighted=4.4150 | Stale=3 | ETA=763.3m +rank0 [2026-11-14 08:33:19,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 08:33:19,235] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 08:33:19,243] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 08:33:19,306] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 08:33:19,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 08:33:20,077] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 08:33:20,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 08:33:20,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 08:33:20,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 08:33:21,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 08:33:21,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 08:33:21,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 08:33:21,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 08:33:22,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 08:33:22,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 08:33:22,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 08:33:22,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 08:33:23,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 08:33:23,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 08:33:23,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 08:33:23,654] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 08:33:23,659] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:33:24,238] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:33:24,593] INFO : Plotted number of elements +rank0 [2026-11-14 08:33:25,888] INFO : Plotted jets +rank0 [2026-11-14 08:33:26,851] INFO : Plotted jet ratio +rank0 [2026-11-14 08:33:27,295] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 08:33:27,295] INFO : Training completed. Total time on device 0: 145.523min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/train.log.0 b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/train.log.0 new file mode 100644 index 0000000000000000000000000000000000000000..5971515862d01ad3cf63514dedfe754de976f553 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/train.log.0 @@ -0,0 +1,1351 @@ +rank0 [2026-11-14 06:07:52,151] INFO : Initializing model with seed=12345; process seed=12345 +rank0 [2026-11-14 06:07:52,151] INFO : configured dtype=torch.bfloat16 for autocast +rank0 [2026-11-14 06:07:52,153] INFO : Instantiating model +rank0 [2026-11-14 06:07:52,175] INFO : MLPF __init__ conv_type=ModelType.ATTENTION num_convs=6 input_encoding=InputEncoding.SPLIT backbone_mode=BackboneMode.SHARED raw_input_dim=19 engineered_input_dim=33 +rank0 [2026-11-14 06:07:52,175] INFO : Initializing split input encoding, elemtypes_nonzero=[1, 2] +rank0 [2026-11-14 06:07:52,177] INFO : Input encoding initialization took 0.00s +rank0 [2026-11-14 06:07:52,177] INFO : nn0 parameters: 140800 +rank0 [2026-11-14 06:07:52,177] INFO : Initializing shared backbone layers, num_convs=6 +rank0 [2026-11-14 06:07:52,177] INFO : Initializing backbone layer 0 +rank0 [2026-11-14 06:07:52,177] INFO : layer backbone_0 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,179] INFO : Initializing backbone layer 1 +rank0 [2026-11-14 06:07:52,179] INFO : layer backbone_1 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,182] INFO : Initializing backbone layer 2 +rank0 [2026-11-14 06:07:52,182] INFO : layer backbone_2 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,185] INFO : Initializing backbone layer 3 +rank0 [2026-11-14 06:07:52,185] INFO : layer backbone_3 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,188] INFO : Initializing backbone layer 4 +rank0 [2026-11-14 06:07:52,188] INFO : layer backbone_4 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,191] INFO : Initializing backbone layer 5 +rank0 [2026-11-14 06:07:52,191] INFO : layer backbone_5 using attention_type=AttentionType.FLASH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,193] INFO : Backbone initialization took 0.02s +rank0 [2026-11-14 06:07:52,193] INFO : backbone parameters: 2374656 +rank0 [2026-11-14 06:07:52,194] INFO : Initializing output DNNs +rank0 [2026-11-14 06:07:52,194] INFO : readout classification_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,197] INFO : readout regression_readout using attention_type=AttentionType.MATH (SimpleMultiheadAttention) +rank0 [2026-11-14 06:07:52,204] INFO : Output DNNs initialization took 0.01s +rank0 [2026-11-14 06:07:52,204] INFO : backbone_mode=BackboneMode.SHARED +rank0 [2026-11-14 06:07:52,204] INFO : jagged_attention enabled=True +rank0 [2026-11-14 06:07:52,204] INFO : flash_attn_varlen enabled=False +rank0 [2026-11-14 06:07:52,204] INFO : task_queries enabled=True +rank0 [2026-11-14 06:07:52,205] INFO : classification_norm parameters: 512 +rank0 [2026-11-14 06:07:52,205] INFO : regression_norm parameters: 512 +rank0 [2026-11-14 06:07:52,205] INFO : classification_query parameters: 256 +rank0 [2026-11-14 06:07:52,205] INFO : regression_query parameters: 256 +rank0 [2026-11-14 06:07:52,205] INFO : classification_readout parameters: 396288 +rank0 [2026-11-14 06:07:52,205] INFO : regression_readout parameters: 396288 +rank0 [2026-11-14 06:07:52,205] INFO : output_mode=elementwise +rank0 [2026-11-14 06:07:52,205] INFO : set_decoder parameters: 0 +rank0 [2026-11-14 06:07:52,205] INFO : nn_binary_particle parameters: 66818 +rank0 [2026-11-14 06:07:52,205] INFO : nn_pid parameters: 67846 +rank0 [2026-11-14 06:07:52,205] INFO : nn_pt parameters: 133122 +rank0 [2026-11-14 06:07:52,205] INFO : nn_eta parameters: 66818 +rank0 [2026-11-14 06:07:52,205] INFO : nn_sin_phi parameters: 66818 +rank0 [2026-11-14 06:07:52,205] INFO : nn_cos_phi parameters: 66818 +rank0 [2026-11-14 06:07:52,205] INFO : nn_energy parameters: 133122 +rank0 [2026-11-14 06:07:52,206] INFO : Total MLPF parameters: 3910930 +rank0 [2026-11-14 06:07:52,206] INFO : MLPF __init__ done +rank0 [2026-11-14 06:07:52,207] INFO : Instantiated model +rank0 [2026-11-14 06:07:52,207] INFO : Moving model to device rank=0 +rank0 [2026-11-14 06:07:52,215] INFO : Moved model to device rank=0 +rank0 [2026-11-14 06:07:52,216] INFO : Created optimizer: Lamb ( +Parameter Group 0 + always_adapt: False + betas: (0.9, 0.999) + bias_correction: True + eps: 1e-06 + grad_averaging: True + lr: 0.0005 + max_grad_norm: 1.0 + trust_clip: False + weight_decay: 0.01 +) +rank0 [2026-11-14 06:07:52,313] INFO : Modules Trainable parameters Non-trainable parameters +0 classification_query 256 0 +1 regression_query 256 0 +2 nn0.0.weight 8448 0 +3 nn0.0.bias 256 0 +4 nn0.2.weight 256 0 +.. ... ... ... +161 nn_energy.nn.1.0.bias 256 0 +162 nn_energy.nn.1.2.weight 256 0 +163 nn_energy.nn.1.2.bias 256 0 +164 nn_energy.nn.1.4.weight 256 0 +165 nn_energy.nn.1.4.bias 1 0 + +[166 rows x 3 columns] +rank0 [2026-11-14 06:07:52,313] INFO : MLPF( + (feature_engineering): Identity() + (input_normalization): InputNormalization() + (feature_bag): FeatureBag() + (nn0): Sequential( + (0): Linear(in_features=33, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=512, bias=True) + ) + (backbone): ModuleList( + (0-5): 6 x PreLnSelfAttentionLayer( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (norm0): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (seq): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + ) + (tracker_backbone): ModuleList() + (calo_backbone): ModuleList() + (classification_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (regression_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (classification_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (regression_readout): TaskQueryAttentionReadout( + (mha): SimpleMultiheadAttention( + (out_proj): Linear(in_features=256, out_features=256, bias=True) + ) + (query_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (key_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (post_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (ffn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): Linear(in_features=256, out_features=256, bias=True) + (3): ELU(alpha=1.0) + ) + (dropout): Dropout(p=0.0, inplace=False) + ) + (nn_binary_particle): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + (nn_pid): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=6, bias=True) + ) + (nn_pt): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (nn_eta): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_sin_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_cos_phi): RegressionOutput( + (nn): Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=2, bias=True) + ) + ) + (nn_energy): RegressionOutput( + (nn): ModuleList( + (0-1): 2 x Sequential( + (0): Linear(in_features=256, out_features=256, bias=True) + (1): ELU(alpha=1.0) + (2): LayerNorm((256,), eps=1e-05, elementwise_affine=True) + (3): Dropout(p=0.0, inplace=False) + (4): Linear(in_features=256, out_features=1, bias=True) + ) + ) + ) + (task_loss_weighter): CalibratedTaskLossWeights() +) +rank0 [2026-11-14 06:07:52,314] INFO : Trainable parameters: 3910930 +rank0 [2026-11-14 06:07:52,314] INFO : Non-trainable parameters: 0 +rank0 [2026-11-14 06:07:52,314] INFO : Total parameters: 3910930 +rank0 [2026-11-14 06:07:52,316] INFO : Modules Trainable parameters Non-trainable parameters + classification_query 256 0 + regression_query 256 0 + nn0.0.weight 8448 0 + nn0.0.bias 256 0 + nn0.2.weight 256 0 + nn0.2.bias 256 0 + nn0.4.weight 131072 0 + nn0.4.bias 512 0 + backbone.0.mha.in_proj_weight 196608 0 + backbone.0.mha.in_proj_bias 768 0 + backbone.0.mha.out_proj.weight 65536 0 + backbone.0.mha.out_proj.bias 256 0 + backbone.0.norm0.weight 256 0 + backbone.0.norm0.bias 256 0 + backbone.0.norm1.weight 256 0 + backbone.0.norm1.bias 256 0 + backbone.0.seq.0.weight 65536 0 + backbone.0.seq.0.bias 256 0 + backbone.0.seq.2.weight 65536 0 + backbone.0.seq.2.bias 256 0 + backbone.1.mha.in_proj_weight 196608 0 + backbone.1.mha.in_proj_bias 768 0 + backbone.1.mha.out_proj.weight 65536 0 + backbone.1.mha.out_proj.bias 256 0 + backbone.1.norm0.weight 256 0 + backbone.1.norm0.bias 256 0 + backbone.1.norm1.weight 256 0 + backbone.1.norm1.bias 256 0 + backbone.1.seq.0.weight 65536 0 + backbone.1.seq.0.bias 256 0 + backbone.1.seq.2.weight 65536 0 + backbone.1.seq.2.bias 256 0 + backbone.2.mha.in_proj_weight 196608 0 + backbone.2.mha.in_proj_bias 768 0 + backbone.2.mha.out_proj.weight 65536 0 + backbone.2.mha.out_proj.bias 256 0 + backbone.2.norm0.weight 256 0 + backbone.2.norm0.bias 256 0 + backbone.2.norm1.weight 256 0 + backbone.2.norm1.bias 256 0 + backbone.2.seq.0.weight 65536 0 + backbone.2.seq.0.bias 256 0 + backbone.2.seq.2.weight 65536 0 + backbone.2.seq.2.bias 256 0 + backbone.3.mha.in_proj_weight 196608 0 + backbone.3.mha.in_proj_bias 768 0 + backbone.3.mha.out_proj.weight 65536 0 + backbone.3.mha.out_proj.bias 256 0 + backbone.3.norm0.weight 256 0 + backbone.3.norm0.bias 256 0 + backbone.3.norm1.weight 256 0 + backbone.3.norm1.bias 256 0 + backbone.3.seq.0.weight 65536 0 + backbone.3.seq.0.bias 256 0 + backbone.3.seq.2.weight 65536 0 + backbone.3.seq.2.bias 256 0 + backbone.4.mha.in_proj_weight 196608 0 + backbone.4.mha.in_proj_bias 768 0 + backbone.4.mha.out_proj.weight 65536 0 + backbone.4.mha.out_proj.bias 256 0 + backbone.4.norm0.weight 256 0 + backbone.4.norm0.bias 256 0 + backbone.4.norm1.weight 256 0 + backbone.4.norm1.bias 256 0 + backbone.4.seq.0.weight 65536 0 + backbone.4.seq.0.bias 256 0 + backbone.4.seq.2.weight 65536 0 + backbone.4.seq.2.bias 256 0 + backbone.5.mha.in_proj_weight 196608 0 + backbone.5.mha.in_proj_bias 768 0 + backbone.5.mha.out_proj.weight 65536 0 + backbone.5.mha.out_proj.bias 256 0 + backbone.5.norm0.weight 256 0 + backbone.5.norm0.bias 256 0 + backbone.5.norm1.weight 256 0 + backbone.5.norm1.bias 256 0 + backbone.5.seq.0.weight 65536 0 + backbone.5.seq.0.bias 256 0 + backbone.5.seq.2.weight 65536 0 + backbone.5.seq.2.bias 256 0 + classification_norm.weight 256 0 + classification_norm.bias 256 0 + regression_norm.weight 256 0 + regression_norm.bias 256 0 + classification_readout.mha.in_proj_weight 196608 0 + classification_readout.mha.in_proj_bias 768 0 +classification_readout.mha.out_proj.weight 65536 0 + classification_readout.mha.out_proj.bias 256 0 + classification_readout.query_norm.weight 256 0 + classification_readout.query_norm.bias 256 0 + classification_readout.key_norm.weight 256 0 + classification_readout.key_norm.bias 256 0 + classification_readout.post_norm.weight 256 0 + classification_readout.post_norm.bias 256 0 + classification_readout.ffn.0.weight 65536 0 + classification_readout.ffn.0.bias 256 0 + classification_readout.ffn.2.weight 65536 0 + classification_readout.ffn.2.bias 256 0 + regression_readout.mha.in_proj_weight 196608 0 + regression_readout.mha.in_proj_bias 768 0 + regression_readout.mha.out_proj.weight 65536 0 + regression_readout.mha.out_proj.bias 256 0 + regression_readout.query_norm.weight 256 0 + regression_readout.query_norm.bias 256 0 + regression_readout.key_norm.weight 256 0 + regression_readout.key_norm.bias 256 0 + regression_readout.post_norm.weight 256 0 + regression_readout.post_norm.bias 256 0 + regression_readout.ffn.0.weight 65536 0 + regression_readout.ffn.0.bias 256 0 + regression_readout.ffn.2.weight 65536 0 + regression_readout.ffn.2.bias 256 0 + nn_binary_particle.0.weight 65536 0 + nn_binary_particle.0.bias 256 0 + nn_binary_particle.2.weight 256 0 + nn_binary_particle.2.bias 256 0 + nn_binary_particle.4.weight 512 0 + nn_binary_particle.4.bias 2 0 + nn_pid.0.weight 65536 0 + nn_pid.0.bias 256 0 + nn_pid.2.weight 256 0 + nn_pid.2.bias 256 0 + nn_pid.4.weight 1536 0 + nn_pid.4.bias 6 0 + nn_pt.nn.0.0.weight 65536 0 + nn_pt.nn.0.0.bias 256 0 + nn_pt.nn.0.2.weight 256 0 + nn_pt.nn.0.2.bias 256 0 + nn_pt.nn.0.4.weight 256 0 + nn_pt.nn.0.4.bias 1 0 + nn_pt.nn.1.0.weight 65536 0 + nn_pt.nn.1.0.bias 256 0 + nn_pt.nn.1.2.weight 256 0 + nn_pt.nn.1.2.bias 256 0 + nn_pt.nn.1.4.weight 256 0 + nn_pt.nn.1.4.bias 1 0 + nn_eta.nn.0.weight 65536 0 + nn_eta.nn.0.bias 256 0 + nn_eta.nn.2.weight 256 0 + nn_eta.nn.2.bias 256 0 + nn_eta.nn.4.weight 512 0 + nn_eta.nn.4.bias 2 0 + nn_sin_phi.nn.0.weight 65536 0 + nn_sin_phi.nn.0.bias 256 0 + nn_sin_phi.nn.2.weight 256 0 + nn_sin_phi.nn.2.bias 256 0 + nn_sin_phi.nn.4.weight 512 0 + nn_sin_phi.nn.4.bias 2 0 + nn_cos_phi.nn.0.weight 65536 0 + nn_cos_phi.nn.0.bias 256 0 + nn_cos_phi.nn.2.weight 256 0 + nn_cos_phi.nn.2.bias 256 0 + nn_cos_phi.nn.4.weight 512 0 + nn_cos_phi.nn.4.bias 2 0 + nn_energy.nn.0.0.weight 65536 0 + nn_energy.nn.0.0.bias 256 0 + nn_energy.nn.0.2.weight 256 0 + nn_energy.nn.0.2.bias 256 0 + nn_energy.nn.0.4.weight 256 0 + nn_energy.nn.0.4.bias 1 0 + nn_energy.nn.1.0.weight 65536 0 + nn_energy.nn.1.0.bias 256 0 + nn_energy.nn.1.2.weight 256 0 + nn_energy.nn.1.2.bias 256 0 + nn_energy.nn.1.4.weight 256 0 + nn_energy.nn.1.4.bias 1 0 +rank0 [2026-11-14 06:07:52,322] INFO : Creating experiment dir /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994 +rank0 [2026-11-14 06:07:52,322] INFO : Model directory /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994 +rank0 [2026-11-14 06:07:52,322] INFO : Getting dataloaders +rank0 [2026-11-14 06:07:52,322] INFO : sample=cld_edm_ttbar_pf split=train split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:07:54,864] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:54,908] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:54,939] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:54,969] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,037] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,208] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,251] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,361] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,405] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,546] INFO : train_dataset: cld_edm_ttbar_pf, 900 +rank0 [2026-11-14 06:07:55,547] INFO : train_dataset sampler_mode=interleaved-shards shuffle=True +rank0 [2026-11-14 06:07:55,547] INFO : Creating WeightedMixIterator with 1 data loaders. +rank0 [2026-11-14 06:07:55,547] INFO : train mix: physical: weight=562 step_share=1.000 batches_per_epoch=562 events_per_step=16 expected_passes_in_500000_steps=889.68 +rank0 [2026-11-14 06:07:55,547] INFO : sample=cld_edm_ttbar_pf split=valid split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:07:55,555] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,562] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,569] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,576] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,583] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,591] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,598] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,605] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,612] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,619] INFO : valid_dataset: cld_edm_ttbar_pf, 25 +rank0 [2026-11-14 06:07:55,619] INFO : valid_dataset sampler_mode=interleaved-shards shuffle=False +rank0 [2026-11-14 06:07:55,619] INFO : Creating InterleavedIterator with 1 data loaders. +rank0 [2026-11-14 06:07:55,678] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:07:55,902] INFO : Got dataloaders +rank0 [2026-11-14 06:07:55,902] INFO : loader split=train rank=0 len=562 +rank0 [2026-11-14 06:07:55,902] INFO : loader split=valid rank=0 len=16 +rank0 [2026-11-14 06:07:55,903] INFO : Starting training from step 1 to 500000 on rank 0 of 1 +rank0 [2026-11-14 06:07:55,910] INFO : Creating train_iterator +rank0 [2026-11-14 06:07:55,910] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:07:55,939] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:07:56,101] INFO : Created train_iterator +rank0 [2026-11-14 06:08:08,355] INFO : Calibrated fixed task weights after 100 steps (100 rank-batches); they will apply from the next training step: losses={'Classification_binary': 7.089501957893372, 'Classification': 0.6290628957748413, 'Regression_pt': 0.5488760888576507, 'Regression_eta': 0.8808271092176437, 'Regression_sin_phi': 0.6515125647187233, 'Regression_cos_phi': 0.6336632177233696, 'Regression_energy': 0.6911959636211396}, weights={'Classification_binary': 1.0, 'Classification': 11.269941329956055, 'Regression_pt': 12.916398048400879, 'Regression_eta': 8.048686981201172, 'Regression_sin_phi': 10.881604194641113, 'Regression_cos_phi': 11.188122749328613, 'Regression_energy': 10.25686264038086} +rank0 [2026-11-14 06:08:59,017] INFO : WeightedMixIterator train: group physical starts epoch 1 +rank0 [2026-11-14 06:08:59,017] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:09:46,863] INFO : Step 1000/500000 rank0 | Train Loss: 12.4256 | Train Loss Unweighted: 5.0118 | Train Loss Components: Classification: 0.0690 | Classification_binary: 4.2982 | Regression_cos_phi: 0.0233 | Regression_energy: 0.2759 | Regression_eta: 0.0322 | Regression_pt: 0.2907 | Regression_sin_phi: 0.0226 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.7781 | Classification_binary: 4.2982 | Regression_cos_phi: 0.2605 | Regression_energy: 2.8295 | Regression_eta: 0.2588 | Regression_pt: 3.7550 | Regression_sin_phi: 0.2456 | LR: 2.01e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5133s | Phase Times: max_pre_step_barrier: 0.0003 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0025 | max_backward: 0.0395 | max_optimizer_step: 0.0655 | max_tensorboard_logging: 0.1274 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2568.0 | uss_MB: 1349.1 | child_rss_MB: 22506.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:10:00,236] INFO : WeightedMixIterator train: group physical starts epoch 2 +rank0 [2026-11-14 06:10:00,236] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:00,380] INFO : WeightedMixIterator train: group physical starts epoch 3 +rank0 [2026-11-14 06:11:00,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:11:36,691] INFO : Step 2000/500000 rank0 | Train Loss: 11.8518 | Train Loss Unweighted: 5.0482 | Train Loss Components: Classification: 0.0526 | Classification_binary: 4.3975 | Regression_cos_phi: 0.0162 | Regression_energy: 0.2653 | Regression_eta: 0.0192 | Regression_pt: 0.2794 | Regression_sin_phi: 0.0180 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.5927 | Classification_binary: 4.3975 | Regression_cos_phi: 0.1813 | Regression_energy: 2.7211 | Regression_eta: 0.1545 | Regression_pt: 3.6084 | Regression_sin_phi: 0.1964 | LR: 2.02e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5335s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.1491 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2569.3 | uss_MB: 1350.4 | child_rss_MB: 22507.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:11:36,714] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 06:12:03,630] INFO : WeightedMixIterator train: group physical starts epoch 4 +rank0 [2026-11-14 06:12:03,630] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:04,365] INFO : WeightedMixIterator train: group physical starts epoch 5 +rank0 [2026-11-14 06:13:04,365] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:13:25,316] INFO : Step 3000/500000 rank0 | Train Loss: 9.9832 | Train Loss Unweighted: 4.6862 | Train Loss Components: Classification: 0.0416 | Classification_binary: 4.1766 | Regression_cos_phi: 0.0136 | Regression_energy: 0.2078 | Regression_eta: 0.0219 | Regression_pt: 0.2126 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4691 | Classification_binary: 4.1766 | Regression_cos_phi: 0.1520 | Regression_energy: 2.1312 | Regression_eta: 0.1765 | Regression_pt: 2.7458 | Regression_sin_phi: 0.1320 | LR: 2.05e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5237s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0027 | max_backward: 0.0408 | max_optimizer_step: 0.0670 | max_tensorboard_logging: 0.1191 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2600.1 | uss_MB: 1381.5 | child_rss_MB: 22509.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:14:05,334] INFO : WeightedMixIterator train: group physical starts epoch 6 +rank0 [2026-11-14 06:14:05,334] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:05,388] INFO : WeightedMixIterator train: group physical starts epoch 7 +rank0 [2026-11-14 06:15:05,388] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:15:12,901] INFO : Step 4000/500000 rank0 | Train Loss: 10.2073 | Train Loss Unweighted: 4.6368 | Train Loss Components: Classification: 0.0447 | Classification_binary: 4.1010 | Regression_cos_phi: 0.0155 | Regression_energy: 0.2175 | Regression_eta: 0.0227 | Regression_pt: 0.2230 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.5037 | Classification_binary: 4.1010 | Regression_cos_phi: 0.1738 | Regression_energy: 2.2313 | Regression_eta: 0.1826 | Regression_pt: 2.8806 | Regression_sin_phi: 0.1342 | LR: 2.08e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4924s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0382 | max_loss: 0.0027 | max_backward: 0.0391 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1161 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2600.2 | uss_MB: 1381.7 | child_rss_MB: 22511.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:15:12,905] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 06:16:06,005] INFO : WeightedMixIterator train: group physical starts epoch 8 +rank0 [2026-11-14 06:16:06,005] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:17:00,595] INFO : Step 5000/500000 rank0 | Train Loss: 9.3765 | Train Loss Unweighted: 4.4695 | Train Loss Components: Classification: 0.0391 | Classification_binary: 3.9994 | Regression_cos_phi: 0.0105 | Regression_energy: 0.1960 | Regression_eta: 0.0157 | Regression_pt: 0.2021 | Regression_sin_phi: 0.0066 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4404 | Classification_binary: 3.9994 | Regression_cos_phi: 0.1170 | Regression_energy: 2.0104 | Regression_eta: 0.1267 | Regression_pt: 2.6103 | Regression_sin_phi: 0.0723 | LR: 2.13e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.4919s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0363 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0660 | max_tensorboard_logging: 0.1166 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2571.1 | uss_MB: 1352.5 | child_rss_MB: 22510.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:17:06,781] INFO : WeightedMixIterator train: group physical starts epoch 9 +rank0 [2026-11-14 06:17:06,781] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:06,981] INFO : WeightedMixIterator train: group physical starts epoch 10 +rank0 [2026-11-14 06:18:06,981] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:18:48,310] INFO : Step 6000/500000 rank0 | Train Loss: 9.8320 | Train Loss Unweighted: 4.5884 | Train Loss Components: Classification: 0.0354 | Classification_binary: 4.0881 | Regression_cos_phi: 0.0122 | Regression_energy: 0.2147 | Regression_eta: 0.0087 | Regression_pt: 0.2168 | Regression_sin_phi: 0.0125 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3992 | Classification_binary: 4.0881 | Regression_cos_phi: 0.1367 | Regression_energy: 2.2018 | Regression_eta: 0.0699 | Regression_pt: 2.7999 | Regression_sin_phi: 0.1363 | LR: 2.19e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5066s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0026 | max_backward: 0.0397 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1312 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2571.6 | uss_MB: 1353.1 | child_rss_MB: 22511.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:18:48,312] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 06:19:07,976] INFO : WeightedMixIterator train: group physical starts epoch 11 +rank0 [2026-11-14 06:19:07,976] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:08,417] INFO : WeightedMixIterator train: group physical starts epoch 12 +rank0 [2026-11-14 06:20:08,417] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:20:36,429] INFO : Step 7000/500000 rank0 | Train Loss: 8.8648 | Train Loss Unweighted: 4.3825 | Train Loss Components: Classification: 0.0387 | Classification_binary: 3.9531 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1811 | Regression_eta: 0.0108 | Regression_pt: 0.1795 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4360 | Classification_binary: 3.9531 | Regression_cos_phi: 0.1089 | Regression_energy: 1.8571 | Regression_eta: 0.0868 | Regression_pt: 2.3179 | Regression_sin_phi: 0.1050 | LR: 2.26e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5142s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0026 | max_backward: 0.0387 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.1242 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2601.6 | uss_MB: 1383.1 | child_rss_MB: 22512.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:21:09,732] INFO : WeightedMixIterator train: group physical starts epoch 13 +rank0 [2026-11-14 06:21:09,732] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:10,263] INFO : WeightedMixIterator train: group physical starts epoch 14 +rank0 [2026-11-14 06:22:10,263] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:22:24,909] INFO : Step 8000/500000 rank0 | Train Loss: 9.2742 | Train Loss Unweighted: 4.4956 | Train Loss Components: Classification: 0.0377 | Classification_binary: 4.0361 | Regression_cos_phi: 0.0167 | Regression_energy: 0.1863 | Regression_eta: 0.0141 | Regression_pt: 0.1842 | Regression_sin_phi: 0.0205 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4253 | Classification_binary: 4.0361 | Regression_cos_phi: 0.1873 | Regression_energy: 1.9104 | Regression_eta: 0.1132 | Regression_pt: 2.3790 | Regression_sin_phi: 0.2230 | LR: 2.34e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5197s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0384 | max_optimizer_step: 0.0761 | max_tensorboard_logging: 0.1252 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2601.7 | uss_MB: 1383.2 | child_rss_MB: 22510.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:22:24,911] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 06:22:24,982] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-2000.pth +rank0 [2026-11-14 06:23:11,328] INFO : WeightedMixIterator train: group physical starts epoch 15 +rank0 [2026-11-14 06:23:11,328] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:11,929] INFO : WeightedMixIterator train: group physical starts epoch 16 +rank0 [2026-11-14 06:24:11,929] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:24:13,205] INFO : Step 9000/500000 rank0 | Train Loss: 8.6153 | Train Loss Unweighted: 4.4547 | Train Loss Components: Classification: 0.0311 | Classification_binary: 4.0567 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1680 | Regression_eta: 0.0103 | Regression_pt: 0.1706 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3505 | Classification_binary: 4.0567 | Regression_cos_phi: 0.1112 | Regression_energy: 1.7232 | Regression_eta: 0.0826 | Regression_pt: 2.2034 | Regression_sin_phi: 0.0876 | LR: 2.43e-05 | DataLoad Time: 0.0002s | Model Forward Time: 0.5107s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0384 | max_optimizer_step: 0.0762 | max_tensorboard_logging: 0.1241 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2570.9 | uss_MB: 1352.4 | child_rss_MB: 22511.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:25:13,380] INFO : WeightedMixIterator train: group physical starts epoch 17 +rank0 [2026-11-14 06:25:13,380] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:26:01,987] INFO : Step 10000/500000 rank0 | Train Loss: 8.4528 | Train Loss Unweighted: 4.0286 | Train Loss Components: Classification: 0.0353 | Classification_binary: 3.6051 | Regression_cos_phi: 0.0087 | Regression_energy: 0.1822 | Regression_eta: 0.0094 | Regression_pt: 0.1788 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3980 | Classification_binary: 3.6051 | Regression_cos_phi: 0.0969 | Regression_energy: 1.8685 | Regression_eta: 0.0756 | Regression_pt: 2.3092 | Regression_sin_phi: 0.0994 | LR: 2.52e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4899s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0365 | max_loss: 0.0026 | max_backward: 0.0392 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1165 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2571.4 | uss_MB: 1352.8 | child_rss_MB: 22513.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 06:26:01,989] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 06:26:02,068] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-4000.pth +rank0 [2026-11-14 06:26:02,068] INFO : Running validation on rank0 +rank0 [2026-11-14 06:26:02,068] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:26:02,068] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:26:02,260] INFO : saved batch0_epoch10000.parquet +rank0 [2026-11-14 06:26:02,471] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:26:02,550] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:26:02,603] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:26:02,658] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:26:02,704] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:26:02,759] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:26:02,813] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:26:02,861] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:26:02,938] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:26:03,015] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:26:03,072] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:26:03,128] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:26:03,176] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:26:03,230] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:26:03,286] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:26:03,337] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:26:03,340] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:26:03,343] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:26:03,344] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:26:03,348] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:26:03,348] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:26:03,352] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:26:03,352] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:26:03,354] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:26:03,356] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:26:03,359] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:26:03,360] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:26:03,364] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:26:03,366] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:26:03,367] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:26:03,370] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:26:04,927] INFO : VALIDATION | Step=10000/500000 | Train Loss=8.4528 | Valid Loss=9.4338 | Train Loss Unweighted=4.0286 | Valid Loss Unweighted=4.4200 | Stale=0 | ETA=889.4m +rank0 [2026-11-14 06:26:04,928] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:26:04,999] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:26:05,006] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:26:05,068] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:26:05,953] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:26:06,210] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:26:06,463] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:26:06,710] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:26:06,956] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:26:07,204] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:26:07,452] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:26:07,723] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:26:07,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:26:08,234] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:26:08,487] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:26:08,731] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:26:08,974] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:26:09,223] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:26:09,492] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:26:09,719] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:26:09,778] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:26:09,784] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:26:10,358] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_10000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:26:11,245] INFO : Plotted number of elements +rank0 [2026-11-14 06:26:12,575] INFO : Plotted jets +rank0 [2026-11-14 06:26:13,959] INFO : Plotted jet ratio +rank0 [2026-11-14 06:26:26,755] INFO : WeightedMixIterator train: group physical starts epoch 18 +rank0 [2026-11-14 06:26:26,755] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:27:27,490] INFO : WeightedMixIterator train: group physical starts epoch 19 +rank0 [2026-11-14 06:27:27,491] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:28:02,584] INFO : Step 11000/500000 rank0 | Train Loss: 10.1441 | Train Loss Unweighted: 4.5251 | Train Loss Components: Classification: 0.0393 | Classification_binary: 3.9750 | Regression_cos_phi: 0.0160 | Regression_energy: 0.2330 | Regression_eta: 0.0407 | Regression_pt: 0.2080 | Regression_sin_phi: 0.0131 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4434 | Classification_binary: 3.9750 | Regression_cos_phi: 0.1786 | Regression_energy: 2.3897 | Regression_eta: 0.3273 | Regression_pt: 2.6871 | Regression_sin_phi: 0.1431 | LR: 2.63e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5187s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0773 | max_tensorboard_logging: 0.1228 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2738.1 | uss_MB: 1527.5 | child_rss_MB: 22514.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:28:28,304] INFO : WeightedMixIterator train: group physical starts epoch 20 +rank0 [2026-11-14 06:28:28,304] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:29:28,448] INFO : WeightedMixIterator train: group physical starts epoch 21 +rank0 [2026-11-14 06:29:28,448] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:29:50,046] INFO : Step 12000/500000 rank0 | Train Loss: 7.9452 | Train Loss Unweighted: 4.0182 | Train Loss Components: Classification: 0.0377 | Classification_binary: 3.6420 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1567 | Regression_eta: 0.0094 | Regression_pt: 0.1560 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4250 | Classification_binary: 3.6420 | Regression_cos_phi: 0.1009 | Regression_energy: 1.6069 | Regression_eta: 0.0756 | Regression_pt: 2.0151 | Regression_sin_phi: 0.0796 | LR: 2.75e-05 | DataLoad Time: 0.0055s | Model Forward Time: 0.5059s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0029 | max_backward: 0.0418 | max_optimizer_step: 0.0692 | max_tensorboard_logging: 0.1243 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2738.1 | uss_MB: 1527.5 | child_rss_MB: 22515.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:29:50,048] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 06:29:50,119] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-6000.pth +rank0 [2026-11-14 06:30:29,133] INFO : WeightedMixIterator train: group physical starts epoch 22 +rank0 [2026-11-14 06:30:29,133] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:31:29,809] INFO : WeightedMixIterator train: group physical starts epoch 23 +rank0 [2026-11-14 06:31:29,809] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:31:38,244] INFO : Step 13000/500000 rank0 | Train Loss: 10.1891 | Train Loss Unweighted: 4.2582 | Train Loss Components: Classification: 0.0384 | Classification_binary: 3.6945 | Regression_cos_phi: 0.0077 | Regression_energy: 0.2480 | Regression_eta: 0.0061 | Regression_pt: 0.2533 | Regression_sin_phi: 0.0103 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4325 | Classification_binary: 3.6945 | Regression_cos_phi: 0.0864 | Regression_energy: 2.5438 | Regression_eta: 0.0491 | Regression_pt: 3.2711 | Regression_sin_phi: 0.1116 | LR: 2.88e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5399s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1320 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2678.7 | uss_MB: 1468.1 | child_rss_MB: 22515.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:32:30,810] INFO : WeightedMixIterator train: group physical starts epoch 24 +rank0 [2026-11-14 06:32:30,810] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:33:26,432] INFO : Step 14000/500000 rank0 | Train Loss: 9.4320 | Train Loss Unweighted: 4.3154 | Train Loss Components: Classification: 0.0387 | Classification_binary: 3.8294 | Regression_cos_phi: 0.0064 | Regression_energy: 0.2101 | Regression_eta: 0.0057 | Regression_pt: 0.2182 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4360 | Classification_binary: 3.8294 | Regression_cos_phi: 0.0719 | Regression_energy: 2.1546 | Regression_eta: 0.0455 | Regression_pt: 2.8186 | Regression_sin_phi: 0.0761 | LR: 3.02e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5209s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0027 | max_backward: 0.0388 | max_optimizer_step: 0.0680 | max_tensorboard_logging: 0.1257 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2679.0 | uss_MB: 1468.4 | child_rss_MB: 22514.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:33:26,434] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:33:26,502] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-8000.pth +rank0 [2026-11-14 06:33:31,863] INFO : WeightedMixIterator train: group physical starts epoch 25 +rank0 [2026-11-14 06:33:31,864] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:34:32,630] INFO : WeightedMixIterator train: group physical starts epoch 26 +rank0 [2026-11-14 06:34:32,630] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:35:15,107] INFO : Step 15000/500000 rank0 | Train Loss: 8.8514 | Train Loss Unweighted: 4.0988 | Train Loss Components: Classification: 0.0334 | Classification_binary: 3.6459 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1946 | Regression_eta: 0.0074 | Regression_pt: 0.1982 | Regression_sin_phi: 0.0093 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3766 | Classification_binary: 3.6459 | Regression_cos_phi: 0.1124 | Regression_energy: 1.9956 | Regression_eta: 0.0594 | Regression_pt: 2.5605 | Regression_sin_phi: 0.1009 | LR: 3.17e-05 | DataLoad Time: 0.0088s | Model Forward Time: 0.5089s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0368 | max_loss: 0.0028 | max_backward: 0.0412 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2681.1 | uss_MB: 1470.5 | child_rss_MB: 22515.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:35:33,728] INFO : WeightedMixIterator train: group physical starts epoch 27 +rank0 [2026-11-14 06:35:33,729] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:36:33,968] INFO : WeightedMixIterator train: group physical starts epoch 28 +rank0 [2026-11-14 06:36:33,969] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:37:02,849] INFO : Step 16000/500000 rank0 | Train Loss: 9.5650 | Train Loss Unweighted: 4.4242 | Train Loss Components: Classification: 0.0337 | Classification_binary: 3.9337 | Regression_cos_phi: 0.0119 | Regression_energy: 0.2033 | Regression_eta: 0.0125 | Regression_pt: 0.2160 | Regression_sin_phi: 0.0131 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3797 | Classification_binary: 3.9337 | Regression_cos_phi: 0.1331 | Regression_energy: 2.0853 | Regression_eta: 0.1006 | Regression_pt: 2.7901 | Regression_sin_phi: 0.1426 | LR: 3.33e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5007s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0378 | max_loss: 0.0028 | max_backward: 0.0397 | max_optimizer_step: 0.0668 | max_tensorboard_logging: 0.1234 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2681.2 | uss_MB: 1470.7 | child_rss_MB: 22518.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:37:02,851] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:37:02,919] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-10000.pth +rank0 [2026-11-14 06:37:34,885] INFO : WeightedMixIterator train: group physical starts epoch 29 +rank0 [2026-11-14 06:37:34,885] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:34,650] INFO : WeightedMixIterator train: group physical starts epoch 30 +rank0 [2026-11-14 06:38:34,650] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:38:49,949] INFO : Step 17000/500000 rank0 | Train Loss: 7.9654 | Train Loss Unweighted: 4.0320 | Train Loss Components: Classification: 0.0361 | Classification_binary: 3.6524 | Regression_cos_phi: 0.0115 | Regression_energy: 0.1453 | Regression_eta: 0.0197 | Regression_pt: 0.1532 | Regression_sin_phi: 0.0138 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4071 | Classification_binary: 3.6524 | Regression_cos_phi: 0.1282 | Regression_energy: 1.4905 | Regression_eta: 0.1588 | Regression_pt: 1.9787 | Regression_sin_phi: 0.1498 | LR: 3.51e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5110s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0342 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0660 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2678.2 | uss_MB: 1467.6 | child_rss_MB: 22517.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:39:35,106] INFO : WeightedMixIterator train: group physical starts epoch 31 +rank0 [2026-11-14 06:39:35,107] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:35,742] INFO : WeightedMixIterator train: group physical starts epoch 32 +rank0 [2026-11-14 06:40:35,743] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:40:37,894] INFO : Step 18000/500000 rank0 | Train Loss: 8.3705 | Train Loss Unweighted: 4.1491 | Train Loss Components: Classification: 0.0396 | Classification_binary: 3.7480 | Regression_cos_phi: 0.0061 | Regression_energy: 0.1718 | Regression_eta: 0.0037 | Regression_pt: 0.1763 | Regression_sin_phi: 0.0035 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4467 | Classification_binary: 3.7480 | Regression_cos_phi: 0.0685 | Regression_energy: 1.7619 | Regression_eta: 0.0301 | Regression_pt: 2.2768 | Regression_sin_phi: 0.0385 | LR: 3.69e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5128s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0004 | max_forward: 0.0370 | max_loss: 0.0028 | max_backward: 0.0398 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2679.0 | uss_MB: 1468.4 | child_rss_MB: 22515.2 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 06:40:37,896] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:40:37,964] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-12000.pth +rank0 [2026-11-14 06:41:36,632] INFO : WeightedMixIterator train: group physical starts epoch 33 +rank0 [2026-11-14 06:41:36,632] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:42:25,821] INFO : Step 19000/500000 rank0 | Train Loss: 8.3097 | Train Loss Unweighted: 4.2182 | Train Loss Components: Classification: 0.0307 | Classification_binary: 3.8275 | Regression_cos_phi: 0.0080 | Regression_energy: 0.1664 | Regression_eta: 0.0076 | Regression_pt: 0.1681 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3456 | Classification_binary: 3.8275 | Regression_cos_phi: 0.0900 | Regression_energy: 1.7067 | Regression_eta: 0.0615 | Regression_pt: 2.1713 | Regression_sin_phi: 0.1072 | LR: 3.88e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5179s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0343 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1312 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2680.9 | uss_MB: 1470.3 | child_rss_MB: 22517.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:42:37,314] INFO : WeightedMixIterator train: group physical starts epoch 34 +rank0 [2026-11-14 06:42:37,314] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:43:37,425] INFO : WeightedMixIterator train: group physical starts epoch 35 +rank0 [2026-11-14 06:43:37,425] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:44:13,282] INFO : Step 20000/500000 rank0 | Train Loss: 9.9121 | Train Loss Unweighted: 4.4119 | Train Loss Components: Classification: 0.0342 | Classification_binary: 3.8874 | Regression_cos_phi: 0.0066 | Regression_energy: 0.2306 | Regression_eta: 0.0097 | Regression_pt: 0.2330 | Regression_sin_phi: 0.0104 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3857 | Classification_binary: 3.8874 | Regression_cos_phi: 0.0738 | Regression_energy: 2.3652 | Regression_eta: 0.0777 | Regression_pt: 3.0094 | Regression_sin_phi: 0.1128 | LR: 4.07e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5235s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1333 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2680.9 | uss_MB: 1470.4 | child_rss_MB: 22515.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 6.3 +rank0 [2026-11-14 06:44:13,284] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:44:13,352] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-14000.pth +rank0 [2026-11-14 06:44:13,353] INFO : Running validation on rank0 +rank0 [2026-11-14 06:44:13,353] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 06:44:13,353] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 06:44:13,503] INFO : saved batch0_epoch20000.parquet +rank0 [2026-11-14 06:44:13,623] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 06:44:13,735] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 06:44:13,804] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 06:44:13,869] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 06:44:13,933] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 06:44:13,999] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 06:44:14,066] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 06:44:14,182] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 06:44:14,295] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 06:44:14,407] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 06:44:14,474] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 06:44:14,539] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 06:44:14,603] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 06:44:14,670] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 06:44:14,737] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 06:44:14,836] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 06:44:14,842] INFO : plotted pt_target histogram +rank0 [2026-11-14 06:44:14,845] INFO : plotted pt_pred histogram +rank0 [2026-11-14 06:44:14,848] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 06:44:14,850] INFO : plotted eta_target histogram +rank0 [2026-11-14 06:44:14,852] INFO : plotted eta_pred histogram +rank0 [2026-11-14 06:44:14,854] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 06:44:14,856] INFO : plotted sphi_target histogram +rank0 [2026-11-14 06:44:14,858] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 06:44:14,860] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 06:44:14,861] INFO : plotted cphi_target histogram +rank0 [2026-11-14 06:44:14,864] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 06:44:14,866] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 06:44:14,868] INFO : plotted energy_target histogram +rank0 [2026-11-14 06:44:14,870] INFO : plotted energy_pred histogram +rank0 [2026-11-14 06:44:14,872] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 06:44:16,418] INFO : VALIDATION | Step=20000/500000 | Train Loss=9.9121 | Valid Loss=9.0020 | Train Loss Unweighted=4.4119 | Valid Loss Unweighted=4.2325 | Stale=0 | ETA=872.2m +rank0 [2026-11-14 06:44:16,418] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 06:44:16,495] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 06:44:16,502] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:16,563] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 06:44:17,106] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 06:44:17,362] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 06:44:17,612] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 06:44:17,866] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 06:44:18,119] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 06:44:18,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 06:44:18,623] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 06:44:18,871] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 06:44:19,121] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 06:44:19,375] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 06:44:19,629] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 06:44:19,883] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 06:44:20,140] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 06:44:20,393] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 06:44:20,642] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 06:44:20,864] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 06:44:20,984] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 06:44:20,990] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:21,563] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_20000/cld_edm_ttbar_pf +rank0 [2026-11-14 06:44:21,916] INFO : Plotted number of elements +rank0 [2026-11-14 06:44:23,223] INFO : Plotted jets +rank0 [2026-11-14 06:44:24,202] INFO : Plotted jet ratio +rank0 [2026-11-14 06:44:49,737] INFO : WeightedMixIterator train: group physical starts epoch 36 +rank0 [2026-11-14 06:44:49,738] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:45:50,689] INFO : WeightedMixIterator train: group physical starts epoch 37 +rank0 [2026-11-14 06:45:50,690] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:46:13,365] INFO : Step 21000/500000 rank0 | Train Loss: 9.4714 | Train Loss Unweighted: 4.2234 | Train Loss Components: Classification: 0.0316 | Classification_binary: 3.7215 | Regression_cos_phi: 0.0127 | Regression_energy: 0.2228 | Regression_eta: 0.0096 | Regression_pt: 0.2156 | Regression_sin_phi: 0.0095 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3563 | Classification_binary: 3.7215 | Regression_cos_phi: 0.1421 | Regression_energy: 2.2855 | Regression_eta: 0.0774 | Regression_pt: 2.7848 | Regression_sin_phi: 0.1038 | LR: 4.28e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5610s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0028 | max_backward: 0.0410 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1387 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2708.0 | uss_MB: 1497.7 | child_rss_MB: 22520.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:46:51,898] INFO : WeightedMixIterator train: group physical starts epoch 38 +rank0 [2026-11-14 06:46:51,898] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:47:51,983] INFO : WeightedMixIterator train: group physical starts epoch 39 +rank0 [2026-11-14 06:47:51,983] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:48:01,188] INFO : Step 22000/500000 rank0 | Train Loss: 8.4509 | Train Loss Unweighted: 3.9942 | Train Loss Components: Classification: 0.0339 | Classification_binary: 3.5687 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1843 | Regression_eta: 0.0088 | Regression_pt: 0.1853 | Regression_sin_phi: 0.0064 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3818 | Classification_binary: 3.5687 | Regression_cos_phi: 0.0758 | Regression_energy: 1.8903 | Regression_eta: 0.0712 | Regression_pt: 2.3931 | Regression_sin_phi: 0.0700 | LR: 4.50e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5141s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0381 | max_loss: 0.0027 | max_backward: 0.0393 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1318 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2708.9 | uss_MB: 1498.5 | child_rss_MB: 22518.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:48:01,189] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:48:01,257] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-16000.pth +rank0 [2026-11-14 06:48:53,283] INFO : WeightedMixIterator train: group physical starts epoch 40 +rank0 [2026-11-14 06:48:53,283] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:49:49,528] INFO : Step 23000/500000 rank0 | Train Loss: 8.0573 | Train Loss Unweighted: 4.0325 | Train Loss Components: Classification: 0.0253 | Classification_binary: 3.6491 | Regression_cos_phi: 0.0051 | Regression_energy: 0.1700 | Regression_eta: 0.0056 | Regression_pt: 0.1707 | Regression_sin_phi: 0.0068 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2847 | Classification_binary: 3.6491 | Regression_cos_phi: 0.0566 | Regression_energy: 1.7437 | Regression_eta: 0.0451 | Regression_pt: 2.2045 | Regression_sin_phi: 0.0736 | LR: 4.73e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5209s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0357 | max_loss: 0.0025 | max_backward: 0.0391 | max_optimizer_step: 0.0679 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2711.9 | uss_MB: 1501.5 | child_rss_MB: 22520.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:49:54,033] INFO : WeightedMixIterator train: group physical starts epoch 41 +rank0 [2026-11-14 06:49:54,034] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:50:54,606] INFO : WeightedMixIterator train: group physical starts epoch 42 +rank0 [2026-11-14 06:50:54,606] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:51:37,789] INFO : Step 24000/500000 rank0 | Train Loss: 8.8659 | Train Loss Unweighted: 4.1369 | Train Loss Components: Classification: 0.0309 | Classification_binary: 3.6863 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1962 | Regression_eta: 0.0069 | Regression_pt: 0.1987 | Regression_sin_phi: 0.0097 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3481 | Classification_binary: 3.6863 | Regression_cos_phi: 0.0912 | Regression_energy: 2.0128 | Regression_eta: 0.0558 | Regression_pt: 2.5660 | Regression_sin_phi: 0.1058 | LR: 4.97e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5182s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0376 | max_loss: 0.0027 | max_backward: 0.0411 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1301 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2712.0 | uss_MB: 1501.6 | child_rss_MB: 22516.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 06:51:37,791] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 06:51:37,861] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-18000.pth +rank0 [2026-11-14 06:51:55,769] INFO : WeightedMixIterator train: group physical starts epoch 43 +rank0 [2026-11-14 06:51:55,769] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:52:55,776] INFO : WeightedMixIterator train: group physical starts epoch 44 +rank0 [2026-11-14 06:52:55,777] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:53:25,832] INFO : Step 25000/500000 rank0 | Train Loss: 8.0612 | Train Loss Unweighted: 4.0299 | Train Loss Components: Classification: 0.0282 | Classification_binary: 3.6457 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1678 | Regression_eta: 0.0069 | Regression_pt: 0.1700 | Regression_sin_phi: 0.0059 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3184 | Classification_binary: 3.6457 | Regression_cos_phi: 0.0605 | Regression_energy: 1.7213 | Regression_eta: 0.0553 | Regression_pt: 2.1962 | Regression_sin_phi: 0.0638 | LR: 5.22e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5114s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0029 | max_backward: 0.0393 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1308 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2708.1 | uss_MB: 1497.7 | child_rss_MB: 22518.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:53:56,928] INFO : WeightedMixIterator train: group physical starts epoch 45 +rank0 [2026-11-14 06:53:56,928] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:54:57,134] INFO : WeightedMixIterator train: group physical starts epoch 46 +rank0 [2026-11-14 06:54:57,134] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:55:13,461] INFO : Step 26000/500000 rank0 | Train Loss: 7.9863 | Train Loss Unweighted: 4.0461 | Train Loss Components: Classification: 0.0293 | Classification_binary: 3.6691 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1562 | Regression_eta: 0.0096 | Regression_pt: 0.1595 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3298 | Classification_binary: 3.6691 | Regression_cos_phi: 0.1276 | Regression_energy: 1.6025 | Regression_eta: 0.0772 | Regression_pt: 2.0601 | Regression_sin_phi: 0.1201 | LR: 5.47e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5296s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0359 | max_loss: 0.0028 | max_backward: 0.0390 | max_optimizer_step: 0.0682 | max_tensorboard_logging: 0.1342 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2709.4 | uss_MB: 1499.1 | child_rss_MB: 22518.8 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 9.9 +rank0 [2026-11-14 06:55:13,463] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 06:55:13,529] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-20000.pth +rank0 [2026-11-14 06:55:57,855] INFO : WeightedMixIterator train: group physical starts epoch 47 +rank0 [2026-11-14 06:55:57,855] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:56:58,477] INFO : WeightedMixIterator train: group physical starts epoch 48 +rank0 [2026-11-14 06:56:58,477] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:57:01,495] INFO : Step 27000/500000 rank0 | Train Loss: 8.7978 | Train Loss Unweighted: 4.1366 | Train Loss Components: Classification: 0.0346 | Classification_binary: 3.6909 | Regression_cos_phi: 0.0097 | Regression_energy: 0.1894 | Regression_eta: 0.0114 | Regression_pt: 0.1925 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3894 | Classification_binary: 3.6909 | Regression_cos_phi: 0.1083 | Regression_energy: 1.9425 | Regression_eta: 0.0921 | Regression_pt: 2.4859 | Regression_sin_phi: 0.0888 | LR: 5.74e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5225s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0359 | max_loss: 0.0025 | max_backward: 0.0387 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1240 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2712.1 | uss_MB: 1501.7 | child_rss_MB: 22519.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:58:00,015] INFO : WeightedMixIterator train: group physical starts epoch 49 +rank0 [2026-11-14 06:58:00,015] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 06:58:50,121] INFO : Step 28000/500000 rank0 | Train Loss: 9.4271 | Train Loss Unweighted: 4.1574 | Train Loss Components: Classification: 0.0305 | Classification_binary: 3.6527 | Regression_cos_phi: 0.0108 | Regression_energy: 0.2160 | Regression_eta: 0.0146 | Regression_pt: 0.2180 | Regression_sin_phi: 0.0147 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3439 | Classification_binary: 3.6527 | Regression_cos_phi: 0.1206 | Regression_energy: 2.2158 | Regression_eta: 0.1177 | Regression_pt: 2.8160 | Regression_sin_phi: 0.1605 | LR: 6.01e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5197s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0378 | max_loss: 0.0027 | max_backward: 0.0395 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1358 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2712.6 | uss_MB: 1502.2 | child_rss_MB: 22518.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 06:58:50,123] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 06:58:50,190] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-22000.pth +rank0 [2026-11-14 06:59:00,999] INFO : WeightedMixIterator train: group physical starts epoch 50 +rank0 [2026-11-14 06:59:00,999] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:00:01,434] INFO : WeightedMixIterator train: group physical starts epoch 51 +rank0 [2026-11-14 07:00:01,434] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:00:38,413] INFO : Step 29000/500000 rank0 | Train Loss: 7.9264 | Train Loss Unweighted: 4.0552 | Train Loss Components: Classification: 0.0274 | Classification_binary: 3.6834 | Regression_cos_phi: 0.0104 | Regression_energy: 0.1554 | Regression_eta: 0.0119 | Regression_pt: 0.1544 | Regression_sin_phi: 0.0122 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3087 | Classification_binary: 3.6834 | Regression_cos_phi: 0.1166 | Regression_energy: 1.5940 | Regression_eta: 0.0962 | Regression_pt: 1.9946 | Regression_sin_phi: 0.1328 | LR: 6.29e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5257s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0399 | max_optimizer_step: 0.0685 | max_tensorboard_logging: 0.1351 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2712.6 | uss_MB: 1502.3 | child_rss_MB: 22517.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:01:02,631] INFO : WeightedMixIterator train: group physical starts epoch 52 +rank0 [2026-11-14 07:01:02,631] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:02:03,209] INFO : WeightedMixIterator train: group physical starts epoch 53 +rank0 [2026-11-14 07:02:03,209] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:02:26,519] INFO : Step 30000/500000 rank0 | Train Loss: 8.3337 | Train Loss Unweighted: 3.8369 | Train Loss Components: Classification: 0.0311 | Classification_binary: 3.4066 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1874 | Regression_eta: 0.0088 | Regression_pt: 0.1832 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3504 | Classification_binary: 3.4066 | Regression_cos_phi: 0.0856 | Regression_energy: 1.9223 | Regression_eta: 0.0707 | Regression_pt: 2.3660 | Regression_sin_phi: 0.1320 | LR: 6.58e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5168s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0026 | max_backward: 0.0388 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1319 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2710.2 | uss_MB: 1499.8 | child_rss_MB: 22517.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:02:26,522] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 07:02:26,590] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-24000.pth +rank0 [2026-11-14 07:02:26,590] INFO : Running validation on rank0 +rank0 [2026-11-14 07:02:26,590] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:02:26,590] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:02:26,739] INFO : saved batch0_epoch30000.parquet +rank0 [2026-11-14 07:02:26,861] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:02:26,974] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:02:27,043] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:02:27,112] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:02:27,178] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:02:27,245] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:02:27,313] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:02:27,413] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:02:27,528] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:02:27,646] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:02:27,714] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:02:27,782] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:02:27,848] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:02:27,914] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:02:27,980] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:02:28,085] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:02:28,092] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:02:28,093] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:02:28,097] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:02:28,099] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:02:28,099] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:02:28,101] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:02:28,103] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:02:28,105] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:02:28,107] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:02:28,109] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:02:28,111] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:02:28,113] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:02:28,117] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:02:28,119] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:02:28,121] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:02:29,680] INFO : VALIDATION | Step=30000/500000 | Train Loss=8.3337 | Valid Loss=8.7145 | Train Loss Unweighted=3.8369 | Valid Loss Unweighted=4.1070 | Stale=0 | ETA=854.8m +rank0 [2026-11-14 07:02:29,681] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:02:29,755] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:02:29,770] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:02:29,868] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:02:30,371] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:02:30,627] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:02:30,879] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:02:31,136] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:02:31,384] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:02:31,628] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:02:31,873] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:02:32,118] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:02:32,366] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:02:32,618] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:02:32,869] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:02:33,124] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:02:33,372] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:02:33,617] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:02:33,867] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:02:34,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:02:34,165] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:02:34,177] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:02:34,741] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_30000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:02:35,092] INFO : Plotted number of elements +rank0 [2026-11-14 07:02:36,396] INFO : Plotted jets +rank0 [2026-11-14 07:02:37,367] INFO : Plotted jet ratio +rank0 [2026-11-14 07:03:15,345] INFO : WeightedMixIterator train: group physical starts epoch 54 +rank0 [2026-11-14 07:03:15,345] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:15,833] INFO : WeightedMixIterator train: group physical starts epoch 55 +rank0 [2026-11-14 07:04:15,833] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:04:25,957] INFO : Step 31000/500000 rank0 | Train Loss: 7.8235 | Train Loss Unweighted: 3.9916 | Train Loss Components: Classification: 0.0297 | Classification_binary: 3.6230 | Regression_cos_phi: 0.0094 | Regression_energy: 0.1548 | Regression_eta: 0.0126 | Regression_pt: 0.1510 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3352 | Classification_binary: 3.6230 | Regression_cos_phi: 0.1047 | Regression_energy: 1.5875 | Regression_eta: 0.1018 | Regression_pt: 1.9505 | Regression_sin_phi: 0.1208 | LR: 6.88e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5150s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0383 | max_loss: 0.0028 | max_backward: 0.0388 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1309 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2849.1 | uss_MB: 1638.8 | child_rss_MB: 22517.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:05:16,665] INFO : WeightedMixIterator train: group physical starts epoch 56 +rank0 [2026-11-14 07:05:16,665] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:06:13,887] INFO : Step 32000/500000 rank0 | Train Loss: 8.4857 | Train Loss Unweighted: 3.6539 | Train Loss Components: Classification: 0.0353 | Classification_binary: 3.1917 | Regression_cos_phi: 0.0150 | Regression_energy: 0.1953 | Regression_eta: 0.0107 | Regression_pt: 0.1957 | Regression_sin_phi: 0.0102 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3977 | Classification_binary: 3.1917 | Regression_cos_phi: 0.1674 | Regression_energy: 2.0032 | Regression_eta: 0.0862 | Regression_pt: 2.5281 | Regression_sin_phi: 0.1114 | LR: 7.19e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5181s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0381 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1272 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2861.9 | uss_MB: 1651.7 | child_rss_MB: 22517.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:06:13,889] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 07:06:13,956] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-26000.pth +rank0 [2026-11-14 07:06:17,618] INFO : WeightedMixIterator train: group physical starts epoch 57 +rank0 [2026-11-14 07:06:17,618] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:07:18,451] INFO : WeightedMixIterator train: group physical starts epoch 58 +rank0 [2026-11-14 07:07:18,451] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:08:01,686] INFO : Step 33000/500000 rank0 | Train Loss: 8.1361 | Train Loss Unweighted: 3.8920 | Train Loss Components: Classification: 0.0299 | Classification_binary: 3.4776 | Regression_cos_phi: 0.0168 | Regression_energy: 0.1781 | Regression_eta: 0.0254 | Regression_pt: 0.1547 | Regression_sin_phi: 0.0096 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3365 | Classification_binary: 3.4776 | Regression_cos_phi: 0.1875 | Regression_energy: 1.8272 | Regression_eta: 0.2047 | Regression_pt: 1.9986 | Regression_sin_phi: 0.1041 | LR: 7.51e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5291s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0340 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0666 | max_tensorboard_logging: 0.1414 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2759.5 | uss_MB: 1559.7 | child_rss_MB: 22518.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:08:18,433] INFO : WeightedMixIterator train: group physical starts epoch 59 +rank0 [2026-11-14 07:08:18,433] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:09:17,980] INFO : WeightedMixIterator train: group physical starts epoch 60 +rank0 [2026-11-14 07:09:17,981] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:09:47,961] INFO : Step 34000/500000 rank0 | Train Loss: 8.3863 | Train Loss Unweighted: 3.8841 | Train Loss Components: Classification: 0.0371 | Classification_binary: 3.4540 | Regression_cos_phi: 0.0106 | Regression_energy: 0.1832 | Regression_eta: 0.0077 | Regression_pt: 0.1821 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4185 | Classification_binary: 3.4540 | Regression_cos_phi: 0.1191 | Regression_energy: 1.8788 | Regression_eta: 0.0617 | Regression_pt: 2.3523 | Regression_sin_phi: 0.1018 | LR: 7.83e-05 | DataLoad Time: 0.0005s | Model Forward Time: 0.5585s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0036 | max_backward: 0.0391 | max_optimizer_step: 0.0702 | max_tensorboard_logging: 0.1425 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2763.1 | uss_MB: 1582.5 | child_rss_MB: 22519.8 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:09:47,965] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 07:09:48,037] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-28000.pth +rank0 [2026-11-14 07:10:17,704] INFO : WeightedMixIterator train: group physical starts epoch 61 +rank0 [2026-11-14 07:10:17,705] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:11:17,612] INFO : WeightedMixIterator train: group physical starts epoch 62 +rank0 [2026-11-14 07:11:17,612] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:11:34,530] INFO : Step 35000/500000 rank0 | Train Loss: 7.8065 | Train Loss Unweighted: 3.7042 | Train Loss Components: Classification: 0.0312 | Classification_binary: 3.3134 | Regression_cos_phi: 0.0085 | Regression_energy: 0.1650 | Regression_eta: 0.0072 | Regression_pt: 0.1718 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3511 | Classification_binary: 3.3134 | Regression_cos_phi: 0.0955 | Regression_energy: 1.6921 | Regression_eta: 0.0581 | Regression_pt: 2.2195 | Regression_sin_phi: 0.0767 | LR: 8.16e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5251s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0348 | max_loss: 0.0025 | max_backward: 0.0381 | max_optimizer_step: 0.0651 | max_tensorboard_logging: 0.1297 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2765.8 | uss_MB: 1598.0 | child_rss_MB: 22517.4 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:12:17,297] INFO : WeightedMixIterator train: group physical starts epoch 63 +rank0 [2026-11-14 07:12:17,298] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:13:16,826] INFO : WeightedMixIterator train: group physical starts epoch 64 +rank0 [2026-11-14 07:13:16,826] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:13:20,633] INFO : Step 36000/500000 rank0 | Train Loss: 7.8293 | Train Loss Unweighted: 3.8267 | Train Loss Components: Classification: 0.0254 | Classification_binary: 3.4447 | Regression_cos_phi: 0.0103 | Regression_energy: 0.1666 | Regression_eta: 0.0062 | Regression_pt: 0.1652 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2868 | Classification_binary: 3.4447 | Regression_cos_phi: 0.1150 | Regression_energy: 1.7087 | Regression_eta: 0.0497 | Regression_pt: 2.1343 | Regression_sin_phi: 0.0901 | LR: 8.50e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5054s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0338 | max_loss: 0.0026 | max_backward: 0.0379 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1252 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2765.8 | uss_MB: 1598.0 | child_rss_MB: 22519.0 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:13:20,636] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 07:13:20,711] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-30000.pth +rank0 [2026-11-14 07:14:16,757] INFO : WeightedMixIterator train: group physical starts epoch 65 +rank0 [2026-11-14 07:14:16,757] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:15:07,179] INFO : Step 37000/500000 rank0 | Train Loss: 8.7545 | Train Loss Unweighted: 3.7327 | Train Loss Components: Classification: 0.0374 | Classification_binary: 3.2523 | Regression_cos_phi: 0.0095 | Regression_energy: 0.2065 | Regression_eta: 0.0127 | Regression_pt: 0.2072 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.4213 | Classification_binary: 3.2523 | Regression_cos_phi: 0.1059 | Regression_energy: 2.1184 | Regression_eta: 0.1026 | Regression_pt: 2.6759 | Regression_sin_phi: 0.0782 | LR: 8.85e-05 | DataLoad Time: 0.0008s | Model Forward Time: 0.5116s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0355 | max_loss: 0.0027 | max_backward: 0.0381 | max_optimizer_step: 0.0657 | max_tensorboard_logging: 0.1323 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2765.8 | uss_MB: 1598.0 | child_rss_MB: 22519.1 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:15:16,900] INFO : WeightedMixIterator train: group physical starts epoch 66 +rank0 [2026-11-14 07:15:16,900] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:16,867] INFO : WeightedMixIterator train: group physical starts epoch 67 +rank0 [2026-11-14 07:16:16,867] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:16:53,866] INFO : Step 38000/500000 rank0 | Train Loss: 8.5823 | Train Loss Unweighted: 3.6180 | Train Loss Components: Classification: 0.0344 | Classification_binary: 3.1446 | Regression_cos_phi: 0.0096 | Regression_energy: 0.2031 | Regression_eta: 0.0084 | Regression_pt: 0.2069 | Regression_sin_phi: 0.0111 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3871 | Classification_binary: 3.1446 | Regression_cos_phi: 0.1070 | Regression_energy: 2.0830 | Regression_eta: 0.0678 | Regression_pt: 2.6723 | Regression_sin_phi: 0.1204 | LR: 9.21e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.4972s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0028 | max_backward: 0.0384 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1271 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2762.9 | uss_MB: 1595.1 | child_rss_MB: 22518.3 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:16:53,869] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 07:16:53,940] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-32000.pth +rank0 [2026-11-14 07:17:16,697] INFO : WeightedMixIterator train: group physical starts epoch 68 +rank0 [2026-11-14 07:17:16,697] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:18:16,450] INFO : WeightedMixIterator train: group physical starts epoch 69 +rank0 [2026-11-14 07:18:16,450] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:18:40,447] INFO : Step 39000/500000 rank0 | Train Loss: 7.4667 | Train Loss Unweighted: 3.7185 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.3604 | Regression_cos_phi: 0.0089 | Regression_energy: 0.1483 | Regression_eta: 0.0073 | Regression_pt: 0.1519 | Regression_sin_phi: 0.0123 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3314 | Classification_binary: 3.3604 | Regression_cos_phi: 0.0995 | Regression_energy: 1.5210 | Regression_eta: 0.0589 | Regression_pt: 1.9619 | Regression_sin_phi: 0.1336 | LR: 9.57e-05 | DataLoad Time: 0.0006s | Model Forward Time: 0.5126s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0356 | max_loss: 0.0027 | max_backward: 0.0385 | max_optimizer_step: 0.0678 | max_tensorboard_logging: 0.1163 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2765.5 | uss_MB: 1597.7 | child_rss_MB: 22519.0 | allocated_MB: 125.7 | reserved_MB: 652.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:19:16,549] INFO : WeightedMixIterator train: group physical starts epoch 70 +rank0 [2026-11-14 07:19:16,549] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:20:16,496] INFO : WeightedMixIterator train: group physical starts epoch 71 +rank0 [2026-11-14 07:20:16,496] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:20:27,314] INFO : Step 40000/500000 rank0 | Train Loss: 7.9207 | Train Loss Unweighted: 3.6823 | Train Loss Components: Classification: 0.0231 | Classification_binary: 3.2785 | Regression_cos_phi: 0.0099 | Regression_energy: 0.1754 | Regression_eta: 0.0071 | Regression_pt: 0.1798 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2603 | Classification_binary: 3.2785 | Regression_cos_phi: 0.1103 | Regression_energy: 1.7985 | Regression_eta: 0.0574 | Regression_pt: 2.3229 | Regression_sin_phi: 0.0929 | LR: 9.94e-05 | DataLoad Time: 0.0007s | Model Forward Time: 0.5025s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0004 | max_forward: 0.0368 | max_loss: 0.0029 | max_backward: 0.0394 | max_optimizer_step: 0.0704 | max_tensorboard_logging: 0.1212 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2766.2 | uss_MB: 1609.2 | child_rss_MB: 22518.8 | allocated_MB: 126.1 | reserved_MB: 652.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:20:27,316] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 07:20:27,387] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-34000.pth +rank0 [2026-11-14 07:20:27,388] INFO : Running validation on rank0 +rank0 [2026-11-14 07:20:27,388] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:20:27,388] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:20:27,538] INFO : saved batch0_epoch40000.parquet +rank0 [2026-11-14 07:20:27,655] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:20:27,764] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:20:27,829] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:20:27,892] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:20:27,954] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:20:28,018] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:20:28,085] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:20:28,179] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:20:28,290] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:20:28,401] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:20:28,468] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:20:28,532] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:20:28,595] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:20:28,659] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:20:28,725] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:20:28,829] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:20:28,836] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:20:28,837] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:20:28,839] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:20:28,842] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:20:28,844] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:20:28,846] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:20:28,848] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:20:28,850] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:20:28,851] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:20:28,854] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:20:28,856] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:20:28,858] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:20:28,860] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:20:28,862] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:20:28,864] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:20:30,359] INFO : VALIDATION | Step=40000/500000 | Train Loss=7.9207 | Valid Loss=8.2393 | Train Loss Unweighted=3.6823 | Valid Loss Unweighted=3.8649 | Stale=0 | ETA=834.6m +rank0 [2026-11-14 07:20:30,360] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:20:30,429] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:20:30,441] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:20:30,514] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:20:31,028] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:20:31,283] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:20:31,535] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:20:31,792] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:20:32,046] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:20:32,299] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:20:32,551] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:20:32,798] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:20:33,047] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:20:33,306] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:20:33,559] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:20:33,812] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:20:34,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:20:34,321] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:20:34,573] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:20:34,795] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:20:34,903] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:20:34,910] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:20:35,480] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_40000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:20:35,842] INFO : Plotted number of elements +rank0 [2026-11-14 07:20:37,167] INFO : Plotted jets +rank0 [2026-11-14 07:20:38,137] INFO : Plotted jet ratio +rank0 [2026-11-14 07:21:28,623] INFO : WeightedMixIterator train: group physical starts epoch 72 +rank0 [2026-11-14 07:21:28,623] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:22:26,785] INFO : Step 41000/500000 rank0 | Train Loss: 7.8426 | Train Loss Unweighted: 3.8514 | Train Loss Components: Classification: 0.0296 | Classification_binary: 3.4686 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1642 | Regression_eta: 0.0101 | Regression_pt: 0.1597 | Regression_sin_phi: 0.0091 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3339 | Classification_binary: 3.4686 | Regression_cos_phi: 0.1122 | Regression_energy: 1.6845 | Regression_eta: 0.0816 | Regression_pt: 2.0626 | Regression_sin_phi: 0.0991 | LR: 1.03e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5447s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0375 | max_loss: 0.0027 | max_backward: 0.0387 | max_optimizer_step: 0.0696 | max_tensorboard_logging: 0.1330 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2796.0 | uss_MB: 1644.2 | child_rss_MB: 22520.5 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:22:29,592] INFO : WeightedMixIterator train: group physical starts epoch 73 +rank0 [2026-11-14 07:22:29,592] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:23:29,545] INFO : WeightedMixIterator train: group physical starts epoch 74 +rank0 [2026-11-14 07:23:29,546] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:24:13,896] INFO : Step 42000/500000 rank0 | Train Loss: 8.6270 | Train Loss Unweighted: 4.0244 | Train Loss Components: Classification: 0.0315 | Classification_binary: 3.5860 | Regression_cos_phi: 0.0064 | Regression_energy: 0.1926 | Regression_eta: 0.0061 | Regression_pt: 0.1934 | Regression_sin_phi: 0.0083 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3555 | Classification_binary: 3.5860 | Regression_cos_phi: 0.0719 | Regression_energy: 1.9758 | Regression_eta: 0.0493 | Regression_pt: 2.4982 | Regression_sin_phi: 0.0903 | LR: 1.07e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5218s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1301 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2830.2 | uss_MB: 1678.4 | child_rss_MB: 22519.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:24:13,898] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 07:24:13,971] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-36000.pth +rank0 [2026-11-14 07:24:29,948] INFO : WeightedMixIterator train: group physical starts epoch 75 +rank0 [2026-11-14 07:24:29,948] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:25:30,212] INFO : WeightedMixIterator train: group physical starts epoch 76 +rank0 [2026-11-14 07:25:30,213] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:26:01,359] INFO : Step 43000/500000 rank0 | Train Loss: 7.3317 | Train Loss Unweighted: 3.6906 | Train Loss Components: Classification: 0.0302 | Classification_binary: 3.3432 | Regression_cos_phi: 0.0069 | Regression_energy: 0.1489 | Regression_eta: 0.0047 | Regression_pt: 0.1480 | Regression_sin_phi: 0.0087 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3398 | Classification_binary: 3.3432 | Regression_cos_phi: 0.0768 | Regression_energy: 1.5273 | Regression_eta: 0.0377 | Regression_pt: 1.9121 | Regression_sin_phi: 0.0948 | LR: 1.11e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5203s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0371 | max_loss: 0.0028 | max_backward: 0.0402 | max_optimizer_step: 0.0708 | max_tensorboard_logging: 0.1313 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2839.3 | uss_MB: 1687.5 | child_rss_MB: 22519.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:26:30,578] INFO : WeightedMixIterator train: group physical starts epoch 77 +rank0 [2026-11-14 07:26:30,578] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:27:30,623] INFO : WeightedMixIterator train: group physical starts epoch 78 +rank0 [2026-11-14 07:27:30,623] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:27:48,653] INFO : Step 44000/500000 rank0 | Train Loss: 8.3316 | Train Loss Unweighted: 3.7067 | Train Loss Components: Classification: 0.0331 | Classification_binary: 3.2637 | Regression_cos_phi: 0.0114 | Regression_energy: 0.1886 | Regression_eta: 0.0110 | Regression_pt: 0.1868 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3733 | Classification_binary: 3.2637 | Regression_cos_phi: 0.1270 | Regression_energy: 1.9341 | Regression_eta: 0.0889 | Regression_pt: 2.4131 | Regression_sin_phi: 0.1315 | LR: 1.15e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5247s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0403 | max_optimizer_step: 0.0675 | max_tensorboard_logging: 0.1304 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2857.4 | uss_MB: 1705.6 | child_rss_MB: 22518.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:27:48,655] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 07:27:48,724] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-38000.pth +rank0 [2026-11-14 07:28:31,433] INFO : WeightedMixIterator train: group physical starts epoch 79 +rank0 [2026-11-14 07:28:31,433] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:31,518] INFO : WeightedMixIterator train: group physical starts epoch 80 +rank0 [2026-11-14 07:29:31,518] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:29:36,183] INFO : Step 45000/500000 rank0 | Train Loss: 7.8993 | Train Loss Unweighted: 3.7366 | Train Loss Components: Classification: 0.0283 | Classification_binary: 3.3398 | Regression_cos_phi: 0.0056 | Regression_energy: 0.1747 | Regression_eta: 0.0066 | Regression_pt: 0.1750 | Regression_sin_phi: 0.0067 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3192 | Classification_binary: 3.3398 | Regression_cos_phi: 0.0625 | Regression_energy: 1.7915 | Regression_eta: 0.0530 | Regression_pt: 2.2610 | Regression_sin_phi: 0.0724 | LR: 1.19e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5054s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0030 | max_backward: 0.0400 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1296 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2873.5 | uss_MB: 1721.8 | child_rss_MB: 22519.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:30:31,551] INFO : WeightedMixIterator train: group physical starts epoch 81 +rank0 [2026-11-14 07:30:31,552] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:31:22,651] INFO : Step 46000/500000 rank0 | Train Loss: 7.9525 | Train Loss Unweighted: 3.9603 | Train Loss Components: Classification: 0.0267 | Classification_binary: 3.5780 | Regression_cos_phi: 0.0128 | Regression_energy: 0.1621 | Regression_eta: 0.0088 | Regression_pt: 0.1604 | Regression_sin_phi: 0.0114 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3013 | Classification_binary: 3.5780 | Regression_cos_phi: 0.1427 | Regression_energy: 1.6629 | Regression_eta: 0.0712 | Regression_pt: 2.0721 | Regression_sin_phi: 0.1244 | LR: 1.23e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4984s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0343 | max_loss: 0.0025 | max_backward: 0.0378 | max_optimizer_step: 0.0655 | max_tensorboard_logging: 0.1198 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2892.2 | uss_MB: 1740.4 | child_rss_MB: 22518.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:31:22,653] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:31:22,733] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-40000.pth +rank0 [2026-11-14 07:31:31,591] INFO : WeightedMixIterator train: group physical starts epoch 82 +rank0 [2026-11-14 07:31:31,591] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:32:30,782] INFO : WeightedMixIterator train: group physical starts epoch 83 +rank0 [2026-11-14 07:32:30,782] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:33:08,516] INFO : Step 47000/500000 rank0 | Train Loss: 7.3969 | Train Loss Unweighted: 3.6368 | Train Loss Components: Classification: 0.0298 | Classification_binary: 3.2787 | Regression_cos_phi: 0.0045 | Regression_energy: 0.1558 | Regression_eta: 0.0052 | Regression_pt: 0.1576 | Regression_sin_phi: 0.0052 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3362 | Classification_binary: 3.2787 | Regression_cos_phi: 0.0504 | Regression_energy: 1.5978 | Regression_eta: 0.0418 | Regression_pt: 2.0355 | Regression_sin_phi: 0.0566 | LR: 1.27e-04 | DataLoad Time: 0.0008s | Model Forward Time: 0.4923s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0380 | max_optimizer_step: 0.0641 | max_tensorboard_logging: 0.1269 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2934.4 | uss_MB: 1782.6 | child_rss_MB: 22518.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 07:33:30,708] INFO : WeightedMixIterator train: group physical starts epoch 84 +rank0 [2026-11-14 07:33:30,708] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:34:30,596] INFO : WeightedMixIterator train: group physical starts epoch 85 +rank0 [2026-11-14 07:34:30,596] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:34:55,519] INFO : Step 48000/500000 rank0 | Train Loss: 7.1036 | Train Loss Unweighted: 3.7263 | Train Loss Components: Classification: 0.0249 | Classification_binary: 3.4035 | Regression_cos_phi: 0.0095 | Regression_energy: 0.1372 | Regression_eta: 0.0068 | Regression_pt: 0.1376 | Regression_sin_phi: 0.0069 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2801 | Classification_binary: 3.4035 | Regression_cos_phi: 0.1058 | Regression_energy: 1.4077 | Regression_eta: 0.0545 | Regression_pt: 1.7767 | Regression_sin_phi: 0.0754 | LR: 1.31e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5050s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0025 | max_backward: 0.0382 | max_optimizer_step: 0.0653 | max_tensorboard_logging: 0.1273 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2936.2 | uss_MB: 1784.4 | child_rss_MB: 22519.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:34:55,520] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:34:55,587] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-42000.pth +rank0 [2026-11-14 07:35:31,096] INFO : WeightedMixIterator train: group physical starts epoch 86 +rank0 [2026-11-14 07:35:31,096] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:36:30,381] INFO : WeightedMixIterator train: group physical starts epoch 87 +rank0 [2026-11-14 07:36:30,381] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:36:42,056] INFO : Step 49000/500000 rank0 | Train Loss: 7.6632 | Train Loss Unweighted: 3.5841 | Train Loss Components: Classification: 0.0206 | Classification_binary: 3.1963 | Regression_cos_phi: 0.0054 | Regression_energy: 0.1732 | Regression_eta: 0.0047 | Regression_pt: 0.1764 | Regression_sin_phi: 0.0075 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2326 | Classification_binary: 3.1963 | Regression_cos_phi: 0.0605 | Regression_energy: 1.7767 | Regression_eta: 0.0378 | Regression_pt: 2.2781 | Regression_sin_phi: 0.0812 | LR: 1.36e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.4944s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0364 | max_loss: 0.0030 | max_backward: 0.0398 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1292 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2936.2 | uss_MB: 1784.4 | child_rss_MB: 22519.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:37:30,568] INFO : WeightedMixIterator train: group physical starts epoch 88 +rank0 [2026-11-14 07:37:30,568] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:38:29,311] INFO : Step 50000/500000 rank0 | Train Loss: 7.6179 | Train Loss Unweighted: 3.5243 | Train Loss Components: Classification: 0.0294 | Classification_binary: 3.1325 | Regression_cos_phi: 0.0123 | Regression_energy: 0.1667 | Regression_eta: 0.0078 | Regression_pt: 0.1635 | Regression_sin_phi: 0.0121 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3312 | Classification_binary: 3.1325 | Regression_cos_phi: 0.1375 | Regression_energy: 1.7097 | Regression_eta: 0.0627 | Regression_pt: 2.1123 | Regression_sin_phi: 0.1321 | LR: 1.40e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5117s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0391 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1298 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2885.3 | uss_MB: 1733.5 | child_rss_MB: 22519.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:38:29,313] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:38:29,382] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-44000.pth +rank0 [2026-11-14 07:38:29,384] INFO : Running validation on rank0 +rank0 [2026-11-14 07:38:29,384] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:38:29,384] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:38:29,538] INFO : saved batch0_epoch50000.parquet +rank0 [2026-11-14 07:38:29,659] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:38:29,779] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:38:29,848] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:38:29,914] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:38:29,979] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:38:30,047] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:38:30,114] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:38:30,214] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:38:30,328] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:38:30,441] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:38:30,507] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:38:30,571] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:38:30,636] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:38:30,701] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:38:30,766] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:38:30,865] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:38:30,872] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:38:30,874] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:38:30,877] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:38:30,879] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:38:30,881] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:38:30,883] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:38:30,885] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:38:30,887] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:38:30,887] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:38:30,891] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:38:30,893] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:38:30,895] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:38:30,897] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:38:30,899] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:38:30,901] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:38:32,400] INFO : VALIDATION | Step=50000/500000 | Train Loss=7.6179 | Valid Loss=8.1213 | Train Loss Unweighted=3.5243 | Valid Loss Unweighted=3.7974 | Stale=0 | ETA=815.5m +rank0 [2026-11-14 07:38:32,400] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:38:32,475] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:38:32,483] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:38:32,563] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:38:33,088] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:38:33,343] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:38:33,597] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:38:33,844] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:38:34,091] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:38:34,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:38:34,593] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:38:34,855] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:38:35,109] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:38:35,364] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:38:35,619] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:38:35,865] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:38:36,112] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:38:36,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:38:36,609] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:38:36,839] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:38:36,904] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:38:36,910] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:38:37,483] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_50000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:38:37,830] INFO : Plotted number of elements +rank0 [2026-11-14 07:38:39,132] INFO : Plotted jets +rank0 [2026-11-14 07:38:40,099] INFO : Plotted jet ratio +rank0 [2026-11-14 07:38:42,446] INFO : WeightedMixIterator train: group physical starts epoch 89 +rank0 [2026-11-14 07:38:42,446] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:39:43,352] INFO : WeightedMixIterator train: group physical starts epoch 90 +rank0 [2026-11-14 07:39:43,352] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:40:28,844] INFO : Step 51000/500000 rank0 | Train Loss: 7.0042 | Train Loss Unweighted: 3.7067 | Train Loss Components: Classification: 0.0271 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0082 | Regression_energy: 0.1224 | Regression_eta: 0.0096 | Regression_pt: 0.1375 | Regression_sin_phi: 0.0098 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3059 | Classification_binary: 3.3922 | Regression_cos_phi: 0.0913 | Regression_energy: 1.2556 | Regression_eta: 0.0775 | Regression_pt: 1.7756 | Regression_sin_phi: 0.1062 | LR: 1.44e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5187s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0345 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0671 | max_tensorboard_logging: 0.1304 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3018.6 | uss_MB: 1866.9 | child_rss_MB: 22519.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:40:44,011] INFO : WeightedMixIterator train: group physical starts epoch 91 +rank0 [2026-11-14 07:40:44,011] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:41:44,335] INFO : WeightedMixIterator train: group physical starts epoch 92 +rank0 [2026-11-14 07:41:44,335] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:42:16,810] INFO : Step 52000/500000 rank0 | Train Loss: 6.2117 | Train Loss Unweighted: 3.4412 | Train Loss Components: Classification: 0.0257 | Classification_binary: 3.1747 | Regression_cos_phi: 0.0067 | Regression_energy: 0.1044 | Regression_eta: 0.0112 | Regression_pt: 0.1091 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2894 | Classification_binary: 3.1747 | Regression_cos_phi: 0.0745 | Regression_energy: 1.0712 | Regression_eta: 0.0901 | Regression_pt: 1.4096 | Regression_sin_phi: 0.1022 | LR: 1.49e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5225s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0004 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1298 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3018.6 | uss_MB: 1866.9 | child_rss_MB: 22528.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:42:16,812] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:42:16,882] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-46000.pth +rank0 [2026-11-14 07:42:45,616] INFO : WeightedMixIterator train: group physical starts epoch 93 +rank0 [2026-11-14 07:42:45,616] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:43:46,734] INFO : WeightedMixIterator train: group physical starts epoch 94 +rank0 [2026-11-14 07:43:46,734] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:44:05,750] INFO : Step 53000/500000 rank0 | Train Loss: 6.4144 | Train Loss Unweighted: 3.4659 | Train Loss Components: Classification: 0.0263 | Classification_binary: 3.1822 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1160 | Regression_eta: 0.0093 | Regression_pt: 0.1135 | Regression_sin_phi: 0.0110 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2962 | Classification_binary: 3.1822 | Regression_cos_phi: 0.0862 | Regression_energy: 1.1901 | Regression_eta: 0.0748 | Regression_pt: 1.4658 | Regression_sin_phi: 0.1192 | LR: 1.53e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5169s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0385 | max_loss: 0.0027 | max_backward: 0.0391 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.1286 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3018.6 | uss_MB: 1866.9 | child_rss_MB: 22525.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:44:47,880] INFO : WeightedMixIterator train: group physical starts epoch 95 +rank0 [2026-11-14 07:44:47,880] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:45:48,653] INFO : WeightedMixIterator train: group physical starts epoch 96 +rank0 [2026-11-14 07:45:48,653] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:45:54,263] INFO : Step 54000/500000 rank0 | Train Loss: 7.0854 | Train Loss Unweighted: 3.5549 | Train Loss Components: Classification: 0.0273 | Classification_binary: 3.2149 | Regression_cos_phi: 0.0070 | Regression_energy: 0.1479 | Regression_eta: 0.0110 | Regression_pt: 0.1387 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3081 | Classification_binary: 3.2149 | Regression_cos_phi: 0.0785 | Regression_energy: 1.5165 | Regression_eta: 0.0889 | Regression_pt: 1.7917 | Regression_sin_phi: 0.0867 | LR: 1.58e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5220s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0350 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0681 | max_tensorboard_logging: 0.1297 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2910.2 | uss_MB: 1758.5 | child_rss_MB: 22524.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:45:54,265] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:45:54,334] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-48000.pth +rank0 [2026-11-14 07:46:49,659] INFO : WeightedMixIterator train: group physical starts epoch 97 +rank0 [2026-11-14 07:46:49,659] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:47:42,525] INFO : Step 55000/500000 rank0 | Train Loss: 6.0413 | Train Loss Unweighted: 3.5286 | Train Loss Components: Classification: 0.0301 | Classification_binary: 3.2859 | Regression_cos_phi: 0.0062 | Regression_energy: 0.0942 | Regression_eta: 0.0104 | Regression_pt: 0.0932 | Regression_sin_phi: 0.0085 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3393 | Classification_binary: 3.2859 | Regression_cos_phi: 0.0694 | Regression_energy: 0.9659 | Regression_eta: 0.0838 | Regression_pt: 1.2040 | Regression_sin_phi: 0.0929 | LR: 1.62e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5126s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0656 | max_tensorboard_logging: 0.1276 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2914.0 | uss_MB: 1762.3 | child_rss_MB: 22525.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 32.3 +rank0 [2026-11-14 07:47:50,697] INFO : WeightedMixIterator train: group physical starts epoch 98 +rank0 [2026-11-14 07:47:50,698] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:48:51,072] INFO : WeightedMixIterator train: group physical starts epoch 99 +rank0 [2026-11-14 07:48:51,072] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:49:30,291] INFO : Step 56000/500000 rank0 | Train Loss: 7.0670 | Train Loss Unweighted: 3.4800 | Train Loss Components: Classification: 0.0251 | Classification_binary: 3.1364 | Regression_cos_phi: 0.0100 | Regression_energy: 0.1440 | Regression_eta: 0.0089 | Regression_pt: 0.1447 | Regression_sin_phi: 0.0109 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2824 | Classification_binary: 3.1364 | Regression_cos_phi: 0.1124 | Regression_energy: 1.4771 | Regression_eta: 0.0720 | Regression_pt: 1.8685 | Regression_sin_phi: 0.1182 | LR: 1.67e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5197s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0386 | max_optimizer_step: 0.0695 | max_tensorboard_logging: 0.1288 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2914.5 | uss_MB: 1762.8 | child_rss_MB: 22532.7 | allocated_MB: 125.7 | reserved_MB: 664.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:49:30,293] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 07:49:30,360] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-50000.pth +rank0 [2026-11-14 07:49:51,825] INFO : WeightedMixIterator train: group physical starts epoch 100 +rank0 [2026-11-14 07:49:51,825] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:50:52,238] INFO : WeightedMixIterator train: group physical starts epoch 101 +rank0 [2026-11-14 07:50:52,239] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:51:18,282] INFO : Step 57000/500000 rank0 | Train Loss: 6.5022 | Train Loss Unweighted: 3.3783 | Train Loss Components: Classification: 0.0261 | Classification_binary: 3.0772 | Regression_cos_phi: 0.0068 | Regression_energy: 0.1239 | Regression_eta: 0.0124 | Regression_pt: 0.1225 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2939 | Classification_binary: 3.0772 | Regression_cos_phi: 0.0766 | Regression_energy: 1.2713 | Regression_eta: 0.1000 | Regression_pt: 1.5829 | Regression_sin_phi: 0.1004 | LR: 1.72e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5169s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0347 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0686 | max_tensorboard_logging: 0.1287 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2915.4 | uss_MB: 1763.7 | child_rss_MB: 22531.1 | allocated_MB: 125.7 | reserved_MB: 664.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:51:53,534] INFO : WeightedMixIterator train: group physical starts epoch 102 +rank0 [2026-11-14 07:51:53,534] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:52:54,529] INFO : WeightedMixIterator train: group physical starts epoch 103 +rank0 [2026-11-14 07:52:54,529] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:53:07,228] INFO : Step 58000/500000 rank0 | Train Loss: 6.4398 | Train Loss Unweighted: 3.4668 | Train Loss Components: Classification: 0.0275 | Classification_binary: 3.1796 | Regression_cos_phi: 0.0057 | Regression_energy: 0.1170 | Regression_eta: 0.0126 | Regression_pt: 0.1138 | Regression_sin_phi: 0.0106 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3095 | Classification_binary: 3.1796 | Regression_cos_phi: 0.0641 | Regression_energy: 1.1999 | Regression_eta: 0.1017 | Regression_pt: 1.4698 | Regression_sin_phi: 0.1154 | LR: 1.76e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5287s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0384 | max_optimizer_step: 0.0762 | max_tensorboard_logging: 0.1310 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2909.7 | uss_MB: 1758.0 | child_rss_MB: 22523.3 | allocated_MB: 125.7 | reserved_MB: 664.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 07:53:07,230] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 07:53:07,299] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-52000.pth +rank0 [2026-11-14 07:53:55,689] INFO : WeightedMixIterator train: group physical starts epoch 104 +rank0 [2026-11-14 07:53:55,689] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:54:55,706] INFO : Step 59000/500000 rank0 | Train Loss: 6.0370 | Train Loss Unweighted: 3.3790 | Train Loss Components: Classification: 0.0280 | Classification_binary: 3.1229 | Regression_cos_phi: 0.0090 | Regression_energy: 0.1027 | Regression_eta: 0.0088 | Regression_pt: 0.0995 | Regression_sin_phi: 0.0080 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3152 | Classification_binary: 3.1229 | Regression_cos_phi: 0.1011 | Regression_energy: 1.0537 | Regression_eta: 0.0709 | Regression_pt: 1.2858 | Regression_sin_phi: 0.0874 | LR: 1.81e-04 | DataLoad Time: 0.0025s | Model Forward Time: 0.5111s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0370 | max_loss: 0.0029 | max_backward: 0.0412 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.1307 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2913.5 | uss_MB: 1761.8 | child_rss_MB: 22518.1 | allocated_MB: 125.7 | reserved_MB: 664.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:54:56,791] INFO : WeightedMixIterator train: group physical starts epoch 105 +rank0 [2026-11-14 07:54:56,791] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:55:57,270] INFO : WeightedMixIterator train: group physical starts epoch 106 +rank0 [2026-11-14 07:55:57,270] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:56:43,955] INFO : Step 60000/500000 rank0 | Train Loss: 6.8870 | Train Loss Unweighted: 3.6523 | Train Loss Components: Classification: 0.0259 | Classification_binary: 3.3404 | Regression_cos_phi: 0.0138 | Regression_energy: 0.1205 | Regression_eta: 0.0147 | Regression_pt: 0.1252 | Regression_sin_phi: 0.0118 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2921 | Classification_binary: 3.3404 | Regression_cos_phi: 0.1549 | Regression_energy: 1.2359 | Regression_eta: 0.1180 | Regression_pt: 1.6176 | Regression_sin_phi: 0.1282 | LR: 1.86e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5317s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0385 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1465 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2913.5 | uss_MB: 1761.8 | child_rss_MB: 22521.8 | allocated_MB: 125.7 | reserved_MB: 664.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 07:56:43,957] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 07:56:44,024] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-54000.pth +rank0 [2026-11-14 07:56:44,025] INFO : Running validation on rank0 +rank0 [2026-11-14 07:56:44,025] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 07:56:44,025] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 07:56:44,174] INFO : saved batch0_epoch60000.parquet +rank0 [2026-11-14 07:56:44,294] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 07:56:44,407] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 07:56:44,475] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 07:56:44,540] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 07:56:44,604] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 07:56:44,670] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 07:56:44,735] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 07:56:44,834] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 07:56:44,947] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 07:56:45,061] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 07:56:45,130] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 07:56:45,195] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 07:56:45,260] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 07:56:45,327] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 07:56:45,394] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 07:56:45,502] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 07:56:45,509] INFO : plotted pt_target histogram +rank0 [2026-11-14 07:56:45,511] INFO : plotted pt_pred histogram +rank0 [2026-11-14 07:56:45,512] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 07:56:45,516] INFO : plotted eta_target histogram +rank0 [2026-11-14 07:56:45,518] INFO : plotted eta_pred histogram +rank0 [2026-11-14 07:56:45,520] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 07:56:45,522] INFO : plotted sphi_target histogram +rank0 [2026-11-14 07:56:45,524] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 07:56:45,526] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 07:56:45,528] INFO : plotted cphi_target histogram +rank0 [2026-11-14 07:56:45,530] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 07:56:45,532] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 07:56:45,534] INFO : plotted energy_target histogram +rank0 [2026-11-14 07:56:45,536] INFO : plotted energy_pred histogram +rank0 [2026-11-14 07:56:45,538] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 07:56:47,007] INFO : VALIDATION | Step=60000/500000 | Train Loss=6.8870 | Valid Loss=8.5005 | Train Loss Unweighted=3.6523 | Valid Loss Unweighted=3.8441 | Stale=1 | ETA=798.2m +rank0 [2026-11-14 07:56:47,007] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 07:56:47,084] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 07:56:47,092] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 07:56:47,155] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 07:56:47,687] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 07:56:47,936] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 07:56:48,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 07:56:48,445] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 07:56:48,696] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 07:56:48,947] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 07:56:49,197] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 07:56:49,444] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 07:56:49,688] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 07:56:49,935] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 07:56:50,191] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 07:56:50,440] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 07:56:50,694] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 07:56:50,946] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 07:56:51,196] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 07:56:51,416] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 07:56:51,522] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 07:56:51,527] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:56:52,106] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_60000/cld_edm_ttbar_pf +rank0 [2026-11-14 07:56:52,462] INFO : Plotted number of elements +rank0 [2026-11-14 07:56:53,758] INFO : Plotted jets +rank0 [2026-11-14 07:56:54,726] INFO : Plotted jet ratio +rank0 [2026-11-14 07:57:09,490] INFO : WeightedMixIterator train: group physical starts epoch 107 +rank0 [2026-11-14 07:57:09,491] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:58:09,932] INFO : WeightedMixIterator train: group physical starts epoch 108 +rank0 [2026-11-14 07:58:09,933] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 07:58:43,054] INFO : Step 61000/500000 rank0 | Train Loss: 6.6271 | Train Loss Unweighted: 3.4380 | Train Loss Components: Classification: 0.0325 | Classification_binary: 3.1311 | Regression_cos_phi: 0.0083 | Regression_energy: 0.1249 | Regression_eta: 0.0109 | Regression_pt: 0.1230 | Regression_sin_phi: 0.0073 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3661 | Classification_binary: 3.1311 | Regression_cos_phi: 0.0933 | Regression_energy: 1.2812 | Regression_eta: 0.0878 | Regression_pt: 1.5887 | Regression_sin_phi: 0.0789 | LR: 1.91e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5399s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0027 | max_backward: 0.0386 | max_optimizer_step: 0.0767 | max_tensorboard_logging: 0.1311 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3041.4 | uss_MB: 1889.7 | child_rss_MB: 22527.2 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 07:59:10,840] INFO : WeightedMixIterator train: group physical starts epoch 109 +rank0 [2026-11-14 07:59:10,841] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:11,195] INFO : WeightedMixIterator train: group physical starts epoch 110 +rank0 [2026-11-14 08:00:11,195] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:00:31,013] INFO : Step 62000/500000 rank0 | Train Loss: 6.0527 | Train Loss Unweighted: 3.1848 | Train Loss Components: Classification: 0.0295 | Classification_binary: 2.9086 | Regression_cos_phi: 0.0062 | Regression_energy: 0.1132 | Regression_eta: 0.0104 | Regression_pt: 0.1107 | Regression_sin_phi: 0.0063 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3324 | Classification_binary: 2.9086 | Regression_cos_phi: 0.0691 | Regression_energy: 1.1608 | Regression_eta: 0.0834 | Regression_pt: 1.4301 | Regression_sin_phi: 0.0683 | LR: 1.95e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5114s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0664 | max_tensorboard_logging: 0.1293 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2934.8 | uss_MB: 1783.1 | child_rss_MB: 22528.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:00:31,015] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 08:00:31,082] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-56000.pth +rank0 [2026-11-14 08:01:12,037] INFO : WeightedMixIterator train: group physical starts epoch 111 +rank0 [2026-11-14 08:01:12,037] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:12,711] INFO : WeightedMixIterator train: group physical starts epoch 112 +rank0 [2026-11-14 08:02:12,711] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:02:19,232] INFO : Step 63000/500000 rank0 | Train Loss: 5.8775 | Train Loss Unweighted: 3.2137 | Train Loss Components: Classification: 0.0245 | Classification_binary: 2.9575 | Regression_cos_phi: 0.0047 | Regression_energy: 0.1084 | Regression_eta: 0.0091 | Regression_pt: 0.1055 | Regression_sin_phi: 0.0040 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2762 | Classification_binary: 2.9575 | Regression_cos_phi: 0.0527 | Regression_energy: 1.1116 | Regression_eta: 0.0730 | Regression_pt: 1.3627 | Regression_sin_phi: 0.0438 | LR: 2.00e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5237s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0354 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0667 | max_tensorboard_logging: 0.1318 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2937.8 | uss_MB: 1786.1 | child_rss_MB: 22530.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:03:13,780] INFO : WeightedMixIterator train: group physical starts epoch 113 +rank0 [2026-11-14 08:03:13,780] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:04:07,568] INFO : Step 64000/500000 rank0 | Train Loss: 5.7416 | Train Loss Unweighted: 3.2509 | Train Loss Components: Classification: 0.0246 | Classification_binary: 3.0107 | Regression_cos_phi: 0.0066 | Regression_energy: 0.1002 | Regression_eta: 0.0079 | Regression_pt: 0.0936 | Regression_sin_phi: 0.0072 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2773 | Classification_binary: 3.0107 | Regression_cos_phi: 0.0740 | Regression_energy: 1.0281 | Regression_eta: 0.0637 | Regression_pt: 1.2091 | Regression_sin_phi: 0.0787 | LR: 2.05e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5220s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.1290 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2937.9 | uss_MB: 1786.2 | child_rss_MB: 22527.6 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:04:07,569] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 08:04:07,636] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-58000.pth +rank0 [2026-11-14 08:04:14,932] INFO : WeightedMixIterator train: group physical starts epoch 114 +rank0 [2026-11-14 08:04:14,932] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:15,545] INFO : WeightedMixIterator train: group physical starts epoch 115 +rank0 [2026-11-14 08:05:15,545] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:05:56,439] INFO : Step 65000/500000 rank0 | Train Loss: 5.4065 | Train Loss Unweighted: 3.1965 | Train Loss Components: Classification: 0.0266 | Classification_binary: 2.9827 | Regression_cos_phi: 0.0062 | Regression_energy: 0.0806 | Regression_eta: 0.0103 | Regression_pt: 0.0809 | Regression_sin_phi: 0.0092 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3001 | Classification_binary: 2.9827 | Regression_cos_phi: 0.0694 | Regression_energy: 0.8263 | Regression_eta: 0.0827 | Regression_pt: 1.0451 | Regression_sin_phi: 0.1002 | LR: 2.10e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5319s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0027 | max_backward: 0.0389 | max_optimizer_step: 0.0690 | max_tensorboard_logging: 0.1323 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2939.0 | uss_MB: 1787.3 | child_rss_MB: 22535.1 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 08:06:17,201] INFO : WeightedMixIterator train: group physical starts epoch 116 +rank0 [2026-11-14 08:06:17,202] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:07:17,793] INFO : WeightedMixIterator train: group physical starts epoch 117 +rank0 [2026-11-14 08:07:17,794] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:07:44,865] INFO : Step 66000/500000 rank0 | Train Loss: 5.6839 | Train Loss Unweighted: 3.0924 | Train Loss Components: Classification: 0.0283 | Classification_binary: 2.8420 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1040 | Regression_eta: 0.0097 | Regression_pt: 0.0965 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.3195 | Classification_binary: 2.8420 | Regression_cos_phi: 0.0710 | Regression_energy: 1.0667 | Regression_eta: 0.0778 | Regression_pt: 1.2461 | Regression_sin_phi: 0.0608 | LR: 2.15e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5303s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0352 | max_loss: 0.0026 | max_backward: 0.0390 | max_optimizer_step: 0.0676 | max_tensorboard_logging: 0.1332 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2934.2 | uss_MB: 1782.5 | child_rss_MB: 22536.4 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:07:44,867] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 08:07:44,937] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-60000.pth +rank0 [2026-11-14 08:08:19,032] INFO : WeightedMixIterator train: group physical starts epoch 118 +rank0 [2026-11-14 08:08:19,032] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:09:19,740] INFO : WeightedMixIterator train: group physical starts epoch 119 +rank0 [2026-11-14 08:09:19,740] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:09:33,355] INFO : Step 67000/500000 rank0 | Train Loss: 5.5839 | Train Loss Unweighted: 3.0728 | Train Loss Components: Classification: 0.0245 | Classification_binary: 2.8314 | Regression_cos_phi: 0.0058 | Regression_energy: 0.0964 | Regression_eta: 0.0084 | Regression_pt: 0.0974 | Regression_sin_phi: 0.0089 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2758 | Classification_binary: 2.8314 | Regression_cos_phi: 0.0644 | Regression_energy: 0.9889 | Regression_eta: 0.0676 | Regression_pt: 1.2584 | Regression_sin_phi: 0.0974 | LR: 2.20e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5248s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0027 | max_backward: 0.0398 | max_optimizer_step: 0.0698 | max_tensorboard_logging: 0.1285 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2937.7 | uss_MB: 1786.0 | child_rss_MB: 22535.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 12.3 +rank0 [2026-11-14 08:10:21,300] INFO : WeightedMixIterator train: group physical starts epoch 120 +rank0 [2026-11-14 08:10:21,300] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:11:22,265] INFO : Step 68000/500000 rank0 | Train Loss: 6.0404 | Train Loss Unweighted: 3.2520 | Train Loss Components: Classification: 0.0266 | Classification_binary: 2.9847 | Regression_cos_phi: 0.0125 | Regression_energy: 0.1061 | Regression_eta: 0.0068 | Regression_pt: 0.1072 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2994 | Classification_binary: 2.9847 | Regression_cos_phi: 0.1403 | Regression_energy: 1.0880 | Regression_eta: 0.0544 | Regression_pt: 1.3842 | Regression_sin_phi: 0.0894 | LR: 2.25e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5222s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0687 | max_tensorboard_logging: 0.1299 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2937.9 | uss_MB: 1786.2 | child_rss_MB: 22535.4 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 7.9 +rank0 [2026-11-14 08:11:22,267] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 08:11:22,337] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-62000.pth +rank0 [2026-11-14 08:11:22,555] INFO : WeightedMixIterator train: group physical starts epoch 121 +rank0 [2026-11-14 08:11:22,555] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:12:23,418] INFO : WeightedMixIterator train: group physical starts epoch 122 +rank0 [2026-11-14 08:12:23,418] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:13:10,966] INFO : Step 69000/500000 rank0 | Train Loss: 6.3528 | Train Loss Unweighted: 3.1760 | Train Loss Components: Classification: 0.0248 | Classification_binary: 2.8714 | Regression_cos_phi: 0.0077 | Regression_energy: 0.1274 | Regression_eta: 0.0089 | Regression_pt: 0.1275 | Regression_sin_phi: 0.0082 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2796 | Classification_binary: 2.8714 | Regression_cos_phi: 0.0864 | Regression_energy: 1.3070 | Regression_eta: 0.0720 | Regression_pt: 1.6467 | Regression_sin_phi: 0.0897 | LR: 2.30e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5126s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0361 | max_loss: 0.0026 | max_backward: 0.0399 | max_optimizer_step: 0.0662 | max_tensorboard_logging: 0.1290 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2937.8 | uss_MB: 1786.1 | child_rss_MB: 22538.9 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 28.3 +rank0 [2026-11-14 08:13:24,569] INFO : WeightedMixIterator train: group physical starts epoch 123 +rank0 [2026-11-14 08:13:24,569] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:14:25,895] INFO : WeightedMixIterator train: group physical starts epoch 124 +rank0 [2026-11-14 08:14:25,895] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:14:59,995] INFO : Step 70000/500000 rank0 | Train Loss: 5.9756 | Train Loss Unweighted: 3.1725 | Train Loss Components: Classification: 0.0208 | Classification_binary: 2.9051 | Regression_cos_phi: 0.0063 | Regression_energy: 0.1114 | Regression_eta: 0.0063 | Regression_pt: 0.1170 | Regression_sin_phi: 0.0057 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2339 | Classification_binary: 2.9051 | Regression_cos_phi: 0.0703 | Regression_energy: 1.1427 | Regression_eta: 0.0506 | Regression_pt: 1.5112 | Regression_sin_phi: 0.0619 | LR: 2.35e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5135s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0353 | max_loss: 0.0026 | max_backward: 0.0386 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.1275 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2934.2 | uss_MB: 1782.5 | child_rss_MB: 22540.7 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 08:14:59,997] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 08:15:00,070] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-64000.pth +rank0 [2026-11-14 08:15:00,071] INFO : Running validation on rank0 +rank0 [2026-11-14 08:15:00,071] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 08:15:00,071] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 08:15:00,220] INFO : saved batch0_epoch70000.parquet +rank0 [2026-11-14 08:15:00,344] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 08:15:00,459] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 08:15:00,529] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 08:15:00,596] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 08:15:00,662] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 08:15:00,729] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 08:15:00,796] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 08:15:00,899] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 08:15:01,014] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 08:15:01,130] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 08:15:01,196] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 08:15:01,261] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 08:15:01,325] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 08:15:01,391] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 08:15:01,455] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 08:15:01,567] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 08:15:01,574] INFO : plotted pt_target histogram +rank0 [2026-11-14 08:15:01,577] INFO : plotted pt_pred histogram +rank0 [2026-11-14 08:15:01,579] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 08:15:01,581] INFO : plotted eta_target histogram +rank0 [2026-11-14 08:15:01,583] INFO : plotted eta_pred histogram +rank0 [2026-11-14 08:15:01,585] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 08:15:01,587] INFO : plotted sphi_target histogram +rank0 [2026-11-14 08:15:01,589] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 08:15:01,591] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 08:15:01,592] INFO : plotted cphi_target histogram +rank0 [2026-11-14 08:15:01,595] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 08:15:01,597] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 08:15:01,599] INFO : plotted energy_target histogram +rank0 [2026-11-14 08:15:01,600] INFO : plotted energy_pred histogram +rank0 [2026-11-14 08:15:01,603] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 08:15:03,074] INFO : VALIDATION | Step=70000/500000 | Train Loss=5.9756 | Valid Loss=9.2647 | Train Loss Unweighted=3.1725 | Valid Loss Unweighted=3.9874 | Stale=2 | ETA=780.9m +rank0 [2026-11-14 08:15:03,074] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 08:15:03,151] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 08:15:03,160] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 08:15:03,224] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 08:15:03,813] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 08:15:04,072] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 08:15:04,328] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 08:15:04,576] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 08:15:04,824] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 08:15:05,071] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 08:15:05,319] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 08:15:05,575] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 08:15:05,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 08:15:06,085] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 08:15:06,340] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 08:15:06,588] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 08:15:06,850] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 08:15:07,099] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 08:15:07,358] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 08:15:07,591] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 08:15:07,660] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 08:15:07,666] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:15:08,247] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_70000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:15:08,595] INFO : Plotted number of elements +rank0 [2026-11-14 08:15:09,901] INFO : Plotted jets +rank0 [2026-11-14 08:15:10,872] INFO : Plotted jet ratio +rank0 [2026-11-14 08:15:38,364] INFO : WeightedMixIterator train: group physical starts epoch 125 +rank0 [2026-11-14 08:15:38,364] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:16:38,972] INFO : WeightedMixIterator train: group physical starts epoch 126 +rank0 [2026-11-14 08:16:38,973] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:16:59,607] INFO : Step 71000/500000 rank0 | Train Loss: 5.5822 | Train Loss Unweighted: 2.9797 | Train Loss Components: Classification: 0.0221 | Classification_binary: 2.7183 | Regression_cos_phi: 0.0086 | Regression_energy: 0.1032 | Regression_eta: 0.0363 | Regression_pt: 0.0861 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2489 | Classification_binary: 2.7183 | Regression_cos_phi: 0.0957 | Regression_energy: 1.0588 | Regression_eta: 0.2924 | Regression_pt: 1.1121 | Regression_sin_phi: 0.0559 | LR: 2.40e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5374s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0025 | max_backward: 0.0383 | max_optimizer_step: 0.0780 | max_tensorboard_logging: 0.1305 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 3070.7 | uss_MB: 1919.0 | child_rss_MB: 22541.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:17:39,854] INFO : WeightedMixIterator train: group physical starts epoch 127 +rank0 [2026-11-14 08:17:39,855] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:18:40,475] INFO : WeightedMixIterator train: group physical starts epoch 128 +rank0 [2026-11-14 08:18:40,476] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:18:47,792] INFO : Step 72000/500000 rank0 | Train Loss: 5.5757 | Train Loss Unweighted: 3.1282 | Train Loss Components: Classification: 0.0252 | Classification_binary: 2.8933 | Regression_cos_phi: 0.0057 | Regression_energy: 0.0969 | Regression_eta: 0.0066 | Regression_pt: 0.0955 | Regression_sin_phi: 0.0051 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2835 | Classification_binary: 2.8933 | Regression_cos_phi: 0.0634 | Regression_energy: 0.9938 | Regression_eta: 0.0532 | Regression_pt: 1.2334 | Regression_sin_phi: 0.0552 | LR: 2.45e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5331s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0390 | max_loss: 0.0028 | max_backward: 0.0404 | max_optimizer_step: 0.0663 | max_tensorboard_logging: 0.1292 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3070.7 | uss_MB: 1919.0 | child_rss_MB: 22540.1 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:18:47,794] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 08:18:47,864] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-66000.pth +rank0 [2026-11-14 08:19:41,935] INFO : WeightedMixIterator train: group physical starts epoch 129 +rank0 [2026-11-14 08:19:41,936] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:20:36,523] INFO : Step 73000/500000 rank0 | Train Loss: 5.3438 | Train Loss Unweighted: 2.9959 | Train Loss Components: Classification: 0.0188 | Classification_binary: 2.7711 | Regression_cos_phi: 0.0081 | Regression_energy: 0.0931 | Regression_eta: 0.0055 | Regression_pt: 0.0937 | Regression_sin_phi: 0.0056 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2117 | Classification_binary: 2.7711 | Regression_cos_phi: 0.0905 | Regression_energy: 0.9549 | Regression_eta: 0.0439 | Regression_pt: 1.2106 | Regression_sin_phi: 0.0610 | LR: 2.50e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5180s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0360 | max_loss: 0.0028 | max_backward: 0.0395 | max_optimizer_step: 0.0677 | max_tensorboard_logging: 0.1289 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 3071.0 | uss_MB: 1919.3 | child_rss_MB: 22546.1 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 11.9 +rank0 [2026-11-14 08:20:42,965] INFO : WeightedMixIterator train: group physical starts epoch 130 +rank0 [2026-11-14 08:20:42,965] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:21:43,635] INFO : WeightedMixIterator train: group physical starts epoch 131 +rank0 [2026-11-14 08:21:43,635] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:22:24,887] INFO : Step 74000/500000 rank0 | Train Loss: 4.9254 | Train Loss Unweighted: 2.6455 | Train Loss Components: Classification: 0.0195 | Classification_binary: 2.4258 | Regression_cos_phi: 0.0084 | Regression_energy: 0.0871 | Regression_eta: 0.0084 | Regression_pt: 0.0869 | Regression_sin_phi: 0.0094 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2197 | Classification_binary: 2.4258 | Regression_cos_phi: 0.0935 | Regression_energy: 0.8939 | Regression_eta: 0.0677 | Regression_pt: 1.1230 | Regression_sin_phi: 0.1018 | LR: 2.55e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5270s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0367 | max_loss: 0.0031 | max_backward: 0.0400 | max_optimizer_step: 0.0699 | max_tensorboard_logging: 0.1416 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2961.2 | uss_MB: 1809.5 | child_rss_MB: 22547.5 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:22:24,890] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 08:22:24,962] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-68000.pth +rank0 [2026-11-14 08:22:44,804] INFO : WeightedMixIterator train: group physical starts epoch 132 +rank0 [2026-11-14 08:22:44,804] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:23:45,473] INFO : WeightedMixIterator train: group physical starts epoch 133 +rank0 [2026-11-14 08:23:45,473] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:24:13,737] INFO : Step 75000/500000 rank0 | Train Loss: 5.3437 | Train Loss Unweighted: 2.7424 | Train Loss Components: Classification: 0.0220 | Classification_binary: 2.4927 | Regression_cos_phi: 0.0053 | Regression_energy: 0.1060 | Regression_eta: 0.0080 | Regression_pt: 0.1044 | Regression_sin_phi: 0.0039 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2483 | Classification_binary: 2.4927 | Regression_cos_phi: 0.0593 | Regression_energy: 1.0872 | Regression_eta: 0.0643 | Regression_pt: 1.3489 | Regression_sin_phi: 0.0430 | LR: 2.60e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5150s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0351 | max_loss: 0.0026 | max_backward: 0.0385 | max_optimizer_step: 0.0659 | max_tensorboard_logging: 0.1297 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2965.1 | uss_MB: 1813.4 | child_rss_MB: 22548.0 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:24:47,084] INFO : WeightedMixIterator train: group physical starts epoch 134 +rank0 [2026-11-14 08:24:47,084] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:25:47,632] INFO : WeightedMixIterator train: group physical starts epoch 135 +rank0 [2026-11-14 08:25:47,632] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:26:02,045] INFO : Step 76000/500000 rank0 | Train Loss: 4.3524 | Train Loss Unweighted: 2.4120 | Train Loss Components: Classification: 0.0231 | Classification_binary: 2.2258 | Regression_cos_phi: 0.0043 | Regression_energy: 0.0768 | Regression_eta: 0.0042 | Regression_pt: 0.0737 | Regression_sin_phi: 0.0041 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2608 | Classification_binary: 2.2258 | Regression_cos_phi: 0.0478 | Regression_energy: 0.7879 | Regression_eta: 0.0338 | Regression_pt: 0.9518 | Regression_sin_phi: 0.0445 | LR: 2.65e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5191s | Phase Times: max_pre_step_barrier: 0.0002 | max_batch_to_device: 0.0003 | max_forward: 0.0349 | max_loss: 0.0026 | max_backward: 0.0383 | max_optimizer_step: 0.0692 | max_tensorboard_logging: 0.1277 | max_tensorboard_flush: 0.0000 | Memory: rss_MB: 2965.2 | uss_MB: 1813.5 | child_rss_MB: 22548.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:26:02,047] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-76000.pth +rank0 [2026-11-14 08:26:02,115] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-70000.pth +rank0 [2026-11-14 08:26:48,636] INFO : WeightedMixIterator train: group physical starts epoch 136 +rank0 [2026-11-14 08:26:48,636] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:27:49,437] INFO : WeightedMixIterator train: group physical starts epoch 137 +rank0 [2026-11-14 08:27:49,437] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:27:50,597] INFO : Step 77000/500000 rank0 | Train Loss: 4.6305 | Train Loss Unweighted: 2.6613 | Train Loss Components: Classification: 0.0233 | Classification_binary: 2.4680 | Regression_cos_phi: 0.0093 | Regression_energy: 0.0743 | Regression_eta: 0.0139 | Regression_pt: 0.0651 | Regression_sin_phi: 0.0074 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2629 | Classification_binary: 2.4680 | Regression_cos_phi: 0.1037 | Regression_energy: 0.7622 | Regression_eta: 0.1122 | Regression_pt: 0.8409 | Regression_sin_phi: 0.0808 | LR: 2.70e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.6132s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0358 | max_loss: 0.0027 | max_backward: 0.0386 | max_optimizer_step: 0.0660 | max_tensorboard_logging: 0.2219 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2966.1 | uss_MB: 1814.4 | child_rss_MB: 22548.5 | allocated_MB: 126.1 | reserved_MB: 648.0 | inactive_split_MB: 7.9 +rank0 [2026-11-14 08:28:50,780] INFO : WeightedMixIterator train: group physical starts epoch 138 +rank0 [2026-11-14 08:28:50,780] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:29:39,065] INFO : Step 78000/500000 rank0 | Train Loss: 5.0095 | Train Loss Unweighted: 2.8530 | Train Loss Components: Classification: 0.0255 | Classification_binary: 2.6452 | Regression_cos_phi: 0.0068 | Regression_energy: 0.0807 | Regression_eta: 0.0072 | Regression_pt: 0.0799 | Regression_sin_phi: 0.0076 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2875 | Classification_binary: 2.6452 | Regression_cos_phi: 0.0765 | Regression_energy: 0.8272 | Regression_eta: 0.0583 | Regression_pt: 1.0322 | Regression_sin_phi: 0.0826 | LR: 2.75e-04 | DataLoad Time: 0.0006s | Model Forward Time: 0.5968s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0393 | max_loss: 0.0027 | max_backward: 0.0389 | max_optimizer_step: 0.0672 | max_tensorboard_logging: 0.2105 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2965.9 | uss_MB: 1814.2 | child_rss_MB: 22551.8 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 8.3 +rank0 [2026-11-14 08:29:39,067] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-78000.pth +rank0 [2026-11-14 08:29:39,136] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-72000.pth +rank0 [2026-11-14 08:29:51,870] INFO : WeightedMixIterator train: group physical starts epoch 139 +rank0 [2026-11-14 08:29:51,870] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:30:52,339] INFO : WeightedMixIterator train: group physical starts epoch 140 +rank0 [2026-11-14 08:30:52,339] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:31:27,346] INFO : Step 79000/500000 rank0 | Train Loss: 4.8829 | Train Loss Unweighted: 2.5650 | Train Loss Components: Classification: 0.0228 | Classification_binary: 2.3422 | Regression_cos_phi: 0.0060 | Regression_energy: 0.0902 | Regression_eta: 0.0069 | Regression_pt: 0.0892 | Regression_sin_phi: 0.0077 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2573 | Classification_binary: 2.3422 | Regression_cos_phi: 0.0672 | Regression_energy: 0.9253 | Regression_eta: 0.0556 | Regression_pt: 1.1516 | Regression_sin_phi: 0.0838 | LR: 2.80e-04 | DataLoad Time: 0.0007s | Model Forward Time: 0.5828s | Phase Times: max_pre_step_barrier: 0.0001 | max_batch_to_device: 0.0003 | max_forward: 0.0373 | max_loss: 0.0027 | max_backward: 0.0396 | max_optimizer_step: 0.0684 | max_tensorboard_logging: 0.1997 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2965.6 | uss_MB: 1813.9 | child_rss_MB: 22554.3 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:31:53,438] INFO : WeightedMixIterator train: group physical starts epoch 141 +rank0 [2026-11-14 08:31:53,439] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:32:54,492] INFO : WeightedMixIterator train: group physical starts epoch 142 +rank0 [2026-11-14 08:32:54,492] INFO : ResumableSampler physical:train loading state: start_index=0 +rank0 [2026-11-14 08:33:16,072] INFO : Step 80000/500000 rank0 | Train Loss: 4.3980 | Train Loss Unweighted: 2.5903 | Train Loss Components: Classification: 0.0192 | Classification_binary: 2.4148 | Regression_cos_phi: 0.0069 | Regression_energy: 0.0643 | Regression_eta: 0.0113 | Regression_pt: 0.0668 | Regression_sin_phi: 0.0070 | Task Weights: Classification: 11.2699 | Classification_binary: 1.0000 | Regression_cos_phi: 11.1881 | Regression_energy: 10.2569 | Regression_eta: 8.0487 | Regression_pt: 12.9164 | Regression_sin_phi: 10.8816 | Task Weighted Losses: Classification: 0.2161 | Classification_binary: 2.4148 | Regression_cos_phi: 0.0768 | Regression_energy: 0.6599 | Regression_eta: 0.0908 | Regression_pt: 0.8631 | Regression_sin_phi: 0.0764 | LR: 2.85e-04 | DataLoad Time: 0.0005s | Model Forward Time: 0.5712s | Phase Times: max_pre_step_barrier: 0.0000 | max_batch_to_device: 0.0003 | max_forward: 0.0346 | max_loss: 0.0025 | max_backward: 0.0387 | max_optimizer_step: 0.0669 | max_tensorboard_logging: 0.1805 | max_tensorboard_flush: 0.0001 | Memory: rss_MB: 2965.7 | uss_MB: 1814.0 | child_rss_MB: 22553.2 | allocated_MB: 125.7 | reserved_MB: 648.0 | inactive_split_MB: 10.3 +rank0 [2026-11-14 08:33:16,074] INFO : saving checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-80000.pth +rank0 [2026-11-14 08:33:16,141] INFO : removing old checkpoint /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/checkpoints/checkpoint-74000.pth +rank0 [2026-11-14 08:33:16,142] INFO : Running validation on rank0 +rank0 [2026-11-14 08:33:16,142] INFO : ResumableSampler physical:valid resetting +rank0 [2026-11-14 08:33:16,142] INFO : ResumableSampler physical:valid loading state: start_index=0 +rank0 [2026-11-14 08:33:16,299] INFO : saved batch0_epoch80000.parquet +rank0 [2026-11-14 08:33:16,421] INFO : plotted energy_elemtype1 +rank0 [2026-11-14 08:33:16,538] INFO : plotted pt_elemtype1 +rank0 [2026-11-14 08:33:16,607] INFO : plotted energy_elemtype1_corr +rank0 [2026-11-14 08:33:16,671] INFO : plotted pt_elemtype1_corr +rank0 [2026-11-14 08:33:16,735] INFO : plotted eta_elemtype1_corr +rank0 [2026-11-14 08:33:16,800] INFO : plotted sphi_elemtype1_corr +rank0 [2026-11-14 08:33:16,865] INFO : plotted cphi_elemtype1_corr +rank0 [2026-11-14 08:33:16,968] INFO : plotted sig_proba_elemtype1 +rank0 [2026-11-14 08:33:17,084] INFO : plotted energy_elemtype2 +rank0 [2026-11-14 08:33:17,196] INFO : plotted pt_elemtype2 +rank0 [2026-11-14 08:33:17,263] INFO : plotted energy_elemtype2_corr +rank0 [2026-11-14 08:33:17,328] INFO : plotted pt_elemtype2_corr +rank0 [2026-11-14 08:33:17,391] INFO : plotted eta_elemtype2_corr +rank0 [2026-11-14 08:33:17,455] INFO : plotted sphi_elemtype2_corr +rank0 [2026-11-14 08:33:17,520] INFO : plotted cphi_elemtype2_corr +rank0 [2026-11-14 08:33:17,622] INFO : plotted sig_proba_elemtype2 +rank0 [2026-11-14 08:33:17,629] INFO : plotted pt_target histogram +rank0 [2026-11-14 08:33:17,631] INFO : plotted pt_pred histogram +rank0 [2026-11-14 08:33:17,633] INFO : plotted pt_ratio histogram +rank0 [2026-11-14 08:33:17,635] INFO : plotted eta_target histogram +rank0 [2026-11-14 08:33:17,637] INFO : plotted eta_pred histogram +rank0 [2026-11-14 08:33:17,639] INFO : plotted eta_ratio histogram +rank0 [2026-11-14 08:33:17,641] INFO : plotted sphi_target histogram +rank0 [2026-11-14 08:33:17,643] INFO : plotted sphi_pred histogram +rank0 [2026-11-14 08:33:17,645] INFO : plotted sphi_ratio histogram +rank0 [2026-11-14 08:33:17,647] INFO : plotted cphi_target histogram +rank0 [2026-11-14 08:33:17,648] INFO : plotted cphi_pred histogram +rank0 [2026-11-14 08:33:17,651] INFO : plotted cphi_ratio histogram +rank0 [2026-11-14 08:33:17,653] INFO : plotted energy_target histogram +rank0 [2026-11-14 08:33:17,654] INFO : plotted energy_pred histogram +rank0 [2026-11-14 08:33:17,658] INFO : plotted energy_ratio histogram +rank0 [2026-11-14 08:33:19,121] INFO : VALIDATION | Step=80000/500000 | Train Loss=4.3980 | Valid Loss=10.1848 | Train Loss Unweighted=2.5903 | Valid Loss Unweighted=4.4150 | Stale=3 | ETA=763.3m +rank0 [2026-11-14 08:33:19,121] INFO : split_configs=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] +rank0 [2026-11-14 08:33:19,235] INFO : test_dataset: cld_edm_ttbar_pf, 250 +rank0 [2026-11-14 08:33:19,243] INFO : Running predictions on cld_edm_ttbar_pf +rank0 [2026-11-14 08:33:19,306] INFO : Worker 0 RLIMIT_NOFILE: soft=131072, hard=131072 +rank0 [2026-11-14 08:33:19,825] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_0.parquet +rank0 [2026-11-14 08:33:20,077] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_1.parquet +rank0 [2026-11-14 08:33:20,326] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_2.parquet +rank0 [2026-11-14 08:33:20,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_3.parquet +rank0 [2026-11-14 08:33:20,828] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_4.parquet +rank0 [2026-11-14 08:33:21,080] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_5.parquet +rank0 [2026-11-14 08:33:21,333] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_6.parquet +rank0 [2026-11-14 08:33:21,577] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_7.parquet +rank0 [2026-11-14 08:33:21,821] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_8.parquet +rank0 [2026-11-14 08:33:22,069] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_9.parquet +rank0 [2026-11-14 08:33:22,318] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_10.parquet +rank0 [2026-11-14 08:33:22,568] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_11.parquet +rank0 [2026-11-14 08:33:22,820] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_12.parquet +rank0 [2026-11-14 08:33:23,074] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_13.parquet +rank0 [2026-11-14 08:33:23,325] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_14.parquet +rank0 [2026-11-14 08:33:23,550] INFO : Saved predictions at /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf/pred_0_15.parquet +rank0 [2026-11-14 08:33:23,654] INFO : Time taken to make predictions on device 0 is: 0.07 min +rank0 [2026-11-14 08:33:23,659] INFO : Loading data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:33:24,238] INFO : Loaded data for plotting from /mnt/home/jpata/particleflow/experiments/pf_fm_experts/cld_hcal_30mm_seed12345_20261114_060723_933994/preds_step_80000/cld_edm_ttbar_pf +rank0 [2026-11-14 08:33:24,593] INFO : Plotted number of elements +rank0 [2026-11-14 08:33:25,888] INFO : Plotted jets +rank0 [2026-11-14 08:33:26,851] INFO : Plotted jet ratio +rank0 [2026-11-14 08:33:27,295] INFO : Stopping early after 3 validation cycles without loss improvement (patience=3) +rank0 [2026-11-14 08:33:27,295] INFO : Training completed. Total time on device 0: 145.523min diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/checkpoints/best.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/checkpoints/best.pth new file mode 100644 index 0000000000000000000000000000000000000000..dc0ab2318fbdfe73ccdafcfc6d877ef57321cd15 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/checkpoints/best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6ae4048f02ee951882e260e360ab653b025776924b3e1f47bee94f436d821f4 +size 47020565 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/checkpoints/checkpoint-500000.pth b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/checkpoints/checkpoint-500000.pth new file mode 100644 index 0000000000000000000000000000000000000000..537d27bd05159c4f8f69025ec802996825a34004 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/checkpoints/checkpoint-500000.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5058a09d1f1e1f1a142c14445e42564c1de9563078b7e036ed016b84b54c2637 +size 47059259 diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_10000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_10000.json new file mode 100644 index 0000000000000000000000000000000000000000..3dced15b47953a4d114426f5c85d44d545717d94 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_10000.json @@ -0,0 +1 @@ +{"epoch": 0.059206981687280565, "train": {"Classification_binary": 3.997204303741455, "Classification": 0.034050602465867996, "Regression_pt": 0.2985426187515259, "Regression_eta": 0.024718793109059334, "Regression_sin_phi": 0.01160099171102047, "Regression_cos_phi": 0.01266014389693737, "Regression_energy": 0.3015877902507782, "Total": 11.229763984680176, "Total_unweighted": 4.680365243926644}, "train_group": {}, "valid": {"Classification_binary": 4.174827088700964, "Classification": 0.033722768438623305, "Regression_pt": 0.2690277505428233, "Regression_eta": 0.02621799073320754, "Regression_sin_phi": 0.013780393499009152, "Regression_cos_phi": 0.016409199288550842, "Regression_energy": 0.2775566628638734, "Total": 10.857400447764295, "Total_unweighted": 4.811541854067051, "metrics/particle/class_1/efficiency": 0.8628789186477661, "metrics/particle/class_1/pid_accuracy": 0.9751701354980469, "metrics/particle/class_2/efficiency": 0.17256011068820953, "metrics/particle/class_2/pid_accuracy": 0.23087431490421295, "metrics/particle/class_3/efficiency": 0.6218844652175903, "metrics/particle/class_3/pid_accuracy": 0.9662496447563171, "metrics/particle/class_4/efficiency": 0.7826961874961853, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 0.9512194991111755, "metrics/particle/class_5/pid_accuracy": 0.0, "metrics/particle/count/bias_mean": 0.7013333439826965, "metrics/particle/count/mae": 4.711999893188477, "metrics/particle/count/prediction_mean": 70.42933654785156, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.12526437640190125, "metrics/particle/event/energy_response_mean": 1.0525747537612915, "metrics/particle/event/met_abs_error": 16.172895431518555, "metrics/particle/event/scalar_pt_relative_abs_error": 0.12441971898078918, "metrics/particle/event/scalar_pt_response_mean": 1.0699385404586792, "metrics/particle/event/vector_pt_closure": 0.1250721663236618, "metrics/particle/matched/delta_eta_abs_mean": 0.026865411549806595, "metrics/particle/matched/delta_phi_abs_mean": 0.020840048789978027, "metrics/particle/matched/delta_r_mean": 0.03766990453004837, "metrics/particle/matched/energy_relative_abs_mean": 0.10422657430171967, "metrics/particle/matched/pid_accuracy": 0.9405714869499207, "metrics/particle/matched/pt_relative_abs_mean": 0.10627909004688263, "metrics/particle/matching/duplicate_fraction": 0.044470105320215225, "metrics/particle/matching/efficiency": 0.7046619057655334, "metrics/particle/matching/f1": 0.7011358737945557, "metrics/particle/matching/purity": 0.6976449489593506, "metrics/particle/matching/target_coverage_dr0p05": 0.530269980430603, "metrics/particle/matching/target_coverage_dr0p10": 0.741012692451477, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6268165707588196, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.785910964012146}, "step_train_time": 5.098946809768677, "step_valid_time": 3.748512327991193, "step_total_time": 796.7313106060028, "elapsed_training_loop_seconds": 796.7313106060028, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0391065218182678, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23717696812391742, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1505, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1317, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1198, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8862718707940781, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8750830564784053, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.880641925777332, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12491694352159469, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8061911170928667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7960132890365449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8010698762955534, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8862718707940781, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0275985674106414, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2428309850334086, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1128, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 945, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 878, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8452593917710197, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8377659574468085, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8414959928762245, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1622340425531915, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7853309481216458, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7783687943262412, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7818343722172751, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8452593917710197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0464584579356169, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.23960407514213067, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 638, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 470, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 444, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7484076433121019, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7366771159874608, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7424960505529226, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.26332288401253917, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7070063694267515, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.6959247648902821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7014218009478673, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7484076433121019, "step_test_time": 9.20368106701062, "step_plot_time": 7.49008720801794, "step_validation_cycle_time": 20.914614783017896} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_100000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_100000.json new file mode 100644 index 0000000000000000000000000000000000000000..49c4ffb3f17581ddb9880db6cc9caa2002723865 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_100000.json @@ -0,0 +1 @@ +{"epoch": 0.5920698168728057, "train": {"Classification_binary": 3.340579032897949, "Classification": 0.022105595096945763, "Regression_pt": 0.16487762331962585, "Regression_eta": 0.01155242882668972, "Regression_sin_phi": 0.011620967648923397, "Regression_cos_phi": 0.008752715773880482, "Regression_energy": 0.17330852150917053, "Total": 7.482454776763916, "Total_unweighted": 3.732796885073185}, "train_group": {}, "valid": {"Classification_binary": 2.9864495460023273, "Classification": 0.021847735060022234, "Regression_pt": 0.17178054566078998, "Regression_eta": 0.01535601945633584, "Regression_sin_phi": 0.008990948504589974, "Regression_cos_phi": 0.009578735270398729, "Regression_energy": 0.17796660484151638, "Total": 7.2543815450465425, "Total_unweighted": 3.3919701347959803, "metrics/particle/class_1/efficiency": 0.913029670715332, "metrics/particle/class_1/pid_accuracy": 0.9806622862815857, "metrics/particle/class_2/efficiency": 0.34912776947021484, "metrics/particle/class_2/pid_accuracy": 0.6758946776390076, "metrics/particle/class_3/efficiency": 0.7476364970207214, "metrics/particle/class_3/pid_accuracy": 0.9812439680099487, "metrics/particle/class_4/efficiency": 0.8370221257209778, "metrics/particle/class_4/pid_accuracy": 0.23317307233810425, "metrics/particle/class_5/efficiency": 0.9471544623374939, "metrics/particle/class_5/pid_accuracy": 0.4806866943836212, "metrics/particle/count/bias_mean": 0.8840000033378601, "metrics/particle/count/mae": 3.7160000801086426, "metrics/particle/count/prediction_mean": 70.61199951171875, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.06366828083992004, "metrics/particle/event/energy_response_mean": 0.9999059438705444, "metrics/particle/event/met_abs_error": 6.787766456604004, "metrics/particle/event/scalar_pt_relative_abs_error": 0.051421571522951126, "metrics/particle/event/scalar_pt_response_mean": 1.0088462829589844, "metrics/particle/event/vector_pt_closure": 0.062452107667922974, "metrics/particle/matched/delta_eta_abs_mean": 0.018986154347658157, "metrics/particle/matched/delta_phi_abs_mean": 0.01614644192159176, "metrics/particle/matched/delta_r_mean": 0.02781393565237522, "metrics/particle/matched/energy_relative_abs_mean": 0.09536106139421463, "metrics/particle/matched/pid_accuracy": 0.9598032236099243, "metrics/particle/matched/pt_relative_abs_mean": 0.09050065279006958, "metrics/particle/matching/duplicate_fraction": 0.038425952196121216, "metrics/particle/matching/efficiency": 0.7968104481697083, "metrics/particle/matching/f1": 0.7917913794517517, "metrics/particle/matching/purity": 0.7868350744247437, "metrics/particle/matching/target_coverage_dr0p05": 0.6912000775337219, "metrics/particle/matching/target_coverage_dr0p10": 0.8318226933479309, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7011626362800598, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8496825695037842}, "step_train_time": 5.1790032386779785, "step_valid_time": 4.36743317701621, "step_total_time": 8069.05729341507, "elapsed_training_loop_seconds": 8069.05729341507, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0099333451047472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10395266100620482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1393, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9374158815612382, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9374158815612382, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9374158815612382, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06258411843876177, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9320323014804845, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9320323014804845, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9320323014804845, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9374158815612382, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023956869787893, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11280991914006289, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1031, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1019, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9221824686940966, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9230080572963295, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9225950782997763, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07699194270367055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9114490161001789, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9122649955237243, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9118568232662192, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9221824686940966, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0122913714471364, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11767130592064579, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 552, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 549, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8789808917197452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8775834658187599, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8782816229116945, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12241653418124006, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8742038216560509, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8728139904610492, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8735083532219571, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8789808917197452, "step_test_time": 9.07752862299094, "step_plot_time": 7.21939657398616, "step_validation_cycle_time": 21.116108147019986} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_110000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_110000.json new file mode 100644 index 0000000000000000000000000000000000000000..6d547afd8a0b6f1b425dde773656693327fd8f30 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_110000.json @@ -0,0 +1 @@ +{"epoch": 0.6512767985600862, "train": {"Classification_binary": 2.8035812377929688, "Classification": 0.02456595003604889, "Regression_pt": 0.1895446628332138, "Regression_eta": 0.010794038884341717, "Regression_sin_phi": 0.013350887224078178, "Regression_cos_phi": 0.00933899451047182, "Regression_energy": 0.18604612350463867, "Total": 7.417658805847168, "Total_unweighted": 3.237221894785762}, "train_group": {}, "valid": {"Classification_binary": 2.880027284013464, "Classification": 0.020666093268293016, "Regression_pt": 0.16931466853364985, "Regression_eta": 0.014987792106384927, "Regression_sin_phi": 0.008722018054191103, "Regression_cos_phi": 0.009390166465272295, "Regression_energy": 0.17463893078743142, "Total": 7.066816614029255, "Total_unweighted": 3.2777469532286867, "metrics/particle/class_1/efficiency": 0.9285827875137329, "metrics/particle/class_1/pid_accuracy": 0.9816697835922241, "metrics/particle/class_2/efficiency": 0.37364450097084045, "metrics/particle/class_2/pid_accuracy": 0.6933754086494446, "metrics/particle/class_3/efficiency": 0.7359207272529602, "metrics/particle/class_3/pid_accuracy": 0.9788554906845093, "metrics/particle/class_4/efficiency": 0.8551307916641235, "metrics/particle/class_4/pid_accuracy": 0.22588235139846802, "metrics/particle/class_5/efficiency": 0.9674796462059021, "metrics/particle/class_5/pid_accuracy": 0.47058823704719543, "metrics/particle/count/bias_mean": 1.993333339691162, "metrics/particle/count/mae": 3.801333427429199, "metrics/particle/count/prediction_mean": 71.7213363647461, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.06091753765940666, "metrics/particle/event/energy_response_mean": 1.0150704383850098, "metrics/particle/event/met_abs_error": 6.859874248504639, "metrics/particle/event/scalar_pt_relative_abs_error": 0.051157400012016296, "metrics/particle/event/scalar_pt_response_mean": 1.0224684476852417, "metrics/particle/event/vector_pt_closure": 0.05838360637426376, "metrics/particle/matched/delta_eta_abs_mean": 0.018231723457574844, "metrics/particle/matched/delta_phi_abs_mean": 0.017431464046239853, "metrics/particle/matched/delta_r_mean": 0.02803419902920723, "metrics/particle/matched/energy_relative_abs_mean": 0.09147097915410995, "metrics/particle/matched/pid_accuracy": 0.959113597869873, "metrics/particle/matched/pt_relative_abs_mean": 0.08672245591878891, "metrics/particle/matching/duplicate_fraction": 0.038370732218027115, "metrics/particle/matching/efficiency": 0.8016100525856018, "metrics/particle/matching/f1": 0.7903136014938354, "metrics/particle/matching/purity": 0.7793310880661011, "metrics/particle/matching/target_coverage_dr0p05": 0.6839911341667175, "metrics/particle/matching/target_coverage_dr0p10": 0.8346527218818665, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7139934301376343, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8588802218437195}, "step_train_time": 5.2201220989227295, "step_valid_time": 4.410414601006778, "step_total_time": 8881.076706409454, "elapsed_training_loop_seconds": 8881.076706409454, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.004863771822044, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0967098241223876, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1389, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9434724091520862, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9396782841823056, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.941571524513096, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06032171581769437, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9347240915208613, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9309651474530831, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9328408327736736, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9434724091520862, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0109476505861918, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11118804255916326, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1113, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1037, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1029, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9275491949910555, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9317160826594789, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9296279695203944, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06828391734052111, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9203935599284436, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9245283018867925, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9224562976243836, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9275491949910555, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.019658863973286, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11947064877124625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 556, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 552, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8853503184713376, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8853503184713376, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8853503184713376, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11464968152866242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8789808917197452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8789808917197452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8789808917197452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8853503184713376, "step_test_time": 9.222192046989221, "step_plot_time": 7.270612215012079, "step_validation_cycle_time": 21.39820631299517} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_120000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_120000.json new file mode 100644 index 0000000000000000000000000000000000000000..05ebf9f2ecaba49ae135c41c0606308b35b1f0a2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_120000.json @@ -0,0 +1 @@ +{"epoch": 0.7104837802473668, "train": {"Classification_binary": 2.6698601245880127, "Classification": 0.019268332049250603, "Regression_pt": 0.15933898091316223, "Regression_eta": 0.009864026680588722, "Regression_sin_phi": 0.0072807134129107, "Regression_cos_phi": 0.006803575437515974, "Regression_energy": 0.16807952523231506, "Total": 6.60374641418457, "Total_unweighted": 3.040495278313756}, "train_group": {}, "valid": {"Classification_binary": 2.7908601152135972, "Classification": 0.019910469968268212, "Regression_pt": 0.16210993300092982, "Regression_eta": 0.01520709661727256, "Regression_sin_phi": 0.008396073858788673, "Regression_cos_phi": 0.009790975362696547, "Regression_energy": 0.16811443897003822, "Total": 6.81818990504488, "Total_unweighted": 3.1743891029915914, "metrics/particle/class_1/efficiency": 0.9221155643463135, "metrics/particle/class_1/pid_accuracy": 0.9791747331619263, "metrics/particle/class_2/efficiency": 0.367986798286438, "metrics/particle/class_2/pid_accuracy": 0.6937860250473022, "metrics/particle/class_3/efficiency": 0.7404894232749939, "metrics/particle/class_3/pid_accuracy": 0.9784361720085144, "metrics/particle/class_4/efficiency": 0.8390341997146606, "metrics/particle/class_4/pid_accuracy": 0.2685851454734802, "metrics/particle/class_5/efficiency": 0.9512194991111755, "metrics/particle/class_5/pid_accuracy": 0.6410256624221802, "metrics/particle/count/bias_mean": 1.0266666412353516, "metrics/particle/count/mae": 3.4213333129882812, "metrics/particle/count/prediction_mean": 70.75466918945312, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.05438509210944176, "metrics/particle/event/energy_response_mean": 0.9928791522979736, "metrics/particle/event/met_abs_error": 6.300232887268066, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04667918011546135, "metrics/particle/event/scalar_pt_response_mean": 1.0066783428192139, "metrics/particle/event/vector_pt_closure": 0.056331854313611984, "metrics/particle/matched/delta_eta_abs_mean": 0.019611170515418053, "metrics/particle/matched/delta_phi_abs_mean": 0.01714453287422657, "metrics/particle/matched/delta_r_mean": 0.028915641829371452, "metrics/particle/matched/energy_relative_abs_mean": 0.0924287885427475, "metrics/particle/matched/pid_accuracy": 0.9592568874359131, "metrics/particle/matched/pt_relative_abs_mean": 0.08623763918876648, "metrics/particle/matching/duplicate_fraction": 0.03599291294813156, "metrics/particle/matching/efficiency": 0.7997361421585083, "metrics/particle/matching/f1": 0.7938915491104126, "metrics/particle/matching/purity": 0.7881317734718323, "metrics/particle/matching/target_coverage_dr0p05": 0.6669535040855408, "metrics/particle/matching/target_coverage_dr0p10": 0.8304650187492371, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7149304151535034, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8575608134269714}, "step_train_time": 5.369901418685913, "step_valid_time": 4.317733417003183, "step_total_time": 9690.957427740097, "elapsed_training_loop_seconds": 9690.957427740097, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0054716656485596, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08797044967539025, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1394, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1382, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9380888290713324, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9412559081701553, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9396697000337041, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0587440918298447, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9300134589502019, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9331532748143146, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9315807212672733, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9380888290713324, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038796273499595, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.11200916174328324, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1115, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1035, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9257602862254025, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9282511210762332, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9270040304523063, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07174887892376682, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9186046511627907, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9210762331838565, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9198387819077474, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9257602862254025, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0096598336323757, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10674267584932706, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 551, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 549, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8773885350318471, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8816, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.879489225857941, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1184, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8742038216560509, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8784, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8762968874700718, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8773885350318471, "step_test_time": 9.14091077100602, "step_plot_time": 7.282645603001583, "step_validation_cycle_time": 21.20235959297861} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_130000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_130000.json new file mode 100644 index 0000000000000000000000000000000000000000..222d2f82e02fbeb7e975a2adca5b0de0484e1888 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_130000.json @@ -0,0 +1 @@ +{"epoch": 0.7696907619346474, "train": {"Classification_binary": 2.9962191581726074, "Classification": 0.0164750125259161, "Regression_pt": 0.13340015709400177, "Regression_eta": 0.020488383248448372, "Regression_sin_phi": 0.010923891328275204, "Regression_cos_phi": 0.007565468084067106, "Regression_energy": 0.14678961038589478, "Total": 6.456090927124023, "Total_unweighted": 3.3318616808392107}, "train_group": {}, "valid": {"Classification_binary": 2.712014543249252, "Classification": 0.01917725040557537, "Regression_pt": 0.15738704356741398, "Regression_eta": 0.013965553425728007, "Regression_sin_phi": 0.00808029479168831, "Regression_cos_phi": 0.008716516038204761, "Regression_energy": 0.16270115020427298, "Total": 6.606673869680851, "Total_unweighted": 3.082042351682136, "metrics/particle/class_1/efficiency": 0.9184653162956238, "metrics/particle/class_1/pid_accuracy": 0.9824614524841309, "metrics/particle/class_2/efficiency": 0.3962753415107727, "metrics/particle/class_2/pid_accuracy": 0.7168352007865906, "metrics/particle/class_3/efficiency": 0.779074490070343, "metrics/particle/class_3/pid_accuracy": 0.9831620454788208, "metrics/particle/class_4/efficiency": 0.7847082614898682, "metrics/particle/class_4/pid_accuracy": 0.31794872879981995, "metrics/particle/class_5/efficiency": 0.9674796462059021, "metrics/particle/class_5/pid_accuracy": 0.6092436909675598, "metrics/particle/count/bias_mean": 0.18666666746139526, "metrics/particle/count/mae": 3.189333438873291, "metrics/particle/count/prediction_mean": 69.91466522216797, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.052415959537029266, "metrics/particle/event/energy_response_mean": 0.998986005783081, "metrics/particle/event/met_abs_error": 5.587776184082031, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04338563606142998, "metrics/particle/event/scalar_pt_response_mean": 1.0043931007385254, "metrics/particle/event/vector_pt_closure": 0.04940575733780861, "metrics/particle/matched/delta_eta_abs_mean": 0.01635008119046688, "metrics/particle/matched/delta_phi_abs_mean": 0.01489644218236208, "metrics/particle/matched/delta_r_mean": 0.024589821696281433, "metrics/particle/matched/energy_relative_abs_mean": 0.09312868863344193, "metrics/particle/matched/pid_accuracy": 0.9641292095184326, "metrics/particle/matched/pt_relative_abs_mean": 0.08817912638187408, "metrics/particle/matching/duplicate_fraction": 0.03459455445408821, "metrics/particle/matching/efficiency": 0.816142737865448, "metrics/particle/matching/f1": 0.8150517344474792, "metrics/particle/matching/purity": 0.8139637112617493, "metrics/particle/matching/target_coverage_dr0p05": 0.7357923984527588, "metrics/particle/matching/target_coverage_dr0p10": 0.8485735058784485, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7182193398475647, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8620926737785339}, "step_train_time": 5.015071868896484, "step_valid_time": 4.384361958014779, "step_total_time": 10496.453817367554, "elapsed_training_loop_seconds": 10496.453817367554, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0070529514413251, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0853574290108493, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1422, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9569313593539704, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9588671611598112, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9578982822499158, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041132838840188805, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9515477792732167, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534726904922455, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.952509262377905, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9569313593539704, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0021040154143752, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09570796977169727, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1114, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1045, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1038, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9347048300536672, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9380610412926391, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9363799283154122, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06193895870736086, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9284436493738819, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9317773788150808, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9301075268817204, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9347048300536672, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0076421068766215, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09768722501758698, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 573, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 571, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9124203821656051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9124203821656051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9124203821656051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0875796178343949, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9092356687898089, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9092356687898089, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9092356687898089, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9124203821656051, "step_test_time": 9.141743908985518, "step_plot_time": 7.395501792023424, "step_validation_cycle_time": 21.387886779994005} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_140000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_140000.json new file mode 100644 index 0000000000000000000000000000000000000000..b3b470e970991419457d0553e3e9c97e2c53ed34 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_140000.json @@ -0,0 +1 @@ +{"epoch": 0.8288977436219279, "train": {"Classification_binary": 2.8640389442443848, "Classification": 0.023316064849495888, "Regression_pt": 0.16093148291110992, "Regression_eta": 0.01218060590326786, "Regression_sin_phi": 0.009882532991468906, "Regression_cos_phi": 0.012806219980120659, "Regression_energy": 0.1583588868379593, "Total": 6.844546318054199, "Total_unweighted": 3.2415147377178073}, "train_group": {}, "valid": {"Classification_binary": 2.672061352019614, "Classification": 0.018555475042221395, "Regression_pt": 0.15190379163052173, "Regression_eta": 0.015367157915805249, "Regression_sin_phi": 0.007867392707378306, "Regression_cos_phi": 0.008563328296580214, "Regression_energy": 0.15784754651658078, "Total": 6.4489875955784575, "Total_unweighted": 3.0321660441287017, "metrics/particle/class_1/efficiency": 0.9293366074562073, "metrics/particle/class_1/pid_accuracy": 0.9846304655075073, "metrics/particle/class_2/efficiency": 0.38967469334602356, "metrics/particle/class_2/pid_accuracy": 0.7059891223907471, "metrics/particle/class_3/efficiency": 0.7732844948768616, "metrics/particle/class_3/pid_accuracy": 0.9837964177131653, "metrics/particle/class_4/efficiency": 0.8309859037399292, "metrics/particle/class_4/pid_accuracy": 0.35593220591545105, "metrics/particle/class_5/efficiency": 0.9715447425842285, "metrics/particle/class_5/pid_accuracy": 0.5230125784873962, "metrics/particle/count/bias_mean": 1.0333333015441895, "metrics/particle/count/mae": 3.315999984741211, "metrics/particle/count/prediction_mean": 70.7613296508789, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.05544077977538109, "metrics/particle/event/energy_response_mean": 1.0146915912628174, "metrics/particle/event/met_abs_error": 6.206449031829834, "metrics/particle/event/scalar_pt_relative_abs_error": 0.048778340220451355, "metrics/particle/event/scalar_pt_response_mean": 1.0177448987960815, "metrics/particle/event/vector_pt_closure": 0.053446948528289795, "metrics/particle/matched/delta_eta_abs_mean": 0.017587533220648766, "metrics/particle/matched/delta_phi_abs_mean": 0.013560422696173191, "metrics/particle/matched/delta_r_mean": 0.02481425739824772, "metrics/particle/matched/energy_relative_abs_mean": 0.09014323353767395, "metrics/particle/matched/pid_accuracy": 0.9649020433425903, "metrics/particle/matched/pt_relative_abs_mean": 0.08400014042854309, "metrics/particle/matching/duplicate_fraction": 0.036253321915864944, "metrics/particle/matching/efficiency": 0.8188580274581909, "metrics/particle/matching/f1": 0.812835156917572, "metrics/particle/matching/purity": 0.8069002032279968, "metrics/particle/matching/target_coverage_dr0p05": 0.7362895607948303, "metrics/particle/matching/target_coverage_dr0p10": 0.8491089344024658, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.727684736251831, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8682499527931213}, "step_train_time": 5.091724872589111, "step_valid_time": 4.368041150999488, "step_total_time": 11306.918060779572, "elapsed_training_loop_seconds": 11306.918060779572, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.007441787943346, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0858651093056253, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1478, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1420, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.955585464333782, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9607577807848444, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9581646423751687, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03924221921515562, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9508748317631225, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.956021650879567, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534412955465587, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.955585464333782, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0067738302079103, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10174381095941132, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1111, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1054, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1053, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9427549194991055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9486948694869487, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9457155675190668, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05130513051305131, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9418604651162791, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9477947794779478, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9448183041722745, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9427549194991055, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.014540275197452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0982533485383481, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 619, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 569, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 566, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9060509554140127, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9192245557350566, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9125902165196471, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08077544426494346, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9012738853503185, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9143780290791599, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9077786688051324, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9060509554140127, "step_test_time": 9.367070214997511, "step_plot_time": 7.289981983980397, "step_validation_cycle_time": 21.49398244099575} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_150000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_150000.json new file mode 100644 index 0000000000000000000000000000000000000000..12fd6abbfda5ccefd17001dec4b18a4f756ac86b --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_150000.json @@ -0,0 +1 @@ +{"epoch": 0.8881047253092085, "train": {"Classification_binary": 2.7727856636047363, "Classification": 0.02030053921043873, "Regression_pt": 0.15400581061840057, "Regression_eta": 0.02083425037562847, "Regression_sin_phi": 0.012181227095425129, "Regression_cos_phi": 0.007515613920986652, "Regression_energy": 0.17884308099746704, "Total": 6.8461079597473145, "Total_unweighted": 3.166466185823083}, "train_group": {}, "valid": {"Classification_binary": 2.626008378698471, "Classification": 0.01819474519567287, "Regression_pt": 0.14882477293623256, "Regression_eta": 0.013968181102833849, "Regression_sin_phi": 0.007843136153322584, "Regression_cos_phi": 0.008475378472754296, "Regression_energy": 0.15401252787163916, "Total": 6.317488650058178, "Total_unweighted": 2.9773271204309264, "metrics/particle/class_1/efficiency": 0.9314394593238831, "metrics/particle/class_1/pid_accuracy": 0.9835150837898254, "metrics/particle/class_2/efficiency": 0.40146157145500183, "metrics/particle/class_2/pid_accuracy": 0.7398707866668701, "metrics/particle/class_3/efficiency": 0.7738725543022156, "metrics/particle/class_3/pid_accuracy": 0.9821136593818665, "metrics/particle/class_4/efficiency": 0.8430583477020264, "metrics/particle/class_4/pid_accuracy": 0.30310261249542236, "metrics/particle/class_5/efficiency": 0.9715447425842285, "metrics/particle/class_5/pid_accuracy": 0.560669481754303, "metrics/particle/count/bias_mean": 1.1506667137145996, "metrics/particle/count/mae": 3.3959999084472656, "metrics/particle/count/prediction_mean": 70.87866973876953, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.052558768540620804, "metrics/particle/event/energy_response_mean": 1.0053068399429321, "metrics/particle/event/met_abs_error": 6.010737419128418, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04663199186325073, "metrics/particle/event/scalar_pt_response_mean": 1.0187280178070068, "metrics/particle/event/vector_pt_closure": 0.05413878709077835, "metrics/particle/matched/delta_eta_abs_mean": 0.015108558349311352, "metrics/particle/matched/delta_phi_abs_mean": 0.014985382556915283, "metrics/particle/matched/delta_r_mean": 0.023757969960570335, "metrics/particle/matched/energy_relative_abs_mean": 0.09066244959831238, "metrics/particle/matched/pid_accuracy": 0.964303195476532, "metrics/particle/matched/pt_relative_abs_mean": 0.0845593512058258, "metrics/particle/matching/duplicate_fraction": 0.03517748787999153, "metrics/particle/matching/efficiency": 0.8211908936500549, "metrics/particle/matching/f1": 0.814470648765564, "metrics/particle/matching/purity": 0.8078594207763672, "metrics/particle/matching/target_coverage_dr0p05": 0.7355629205703735, "metrics/particle/matching/target_coverage_dr0p10": 0.8580388426780701, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7245486974716187, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.865649402141571}, "step_train_time": 5.297961711883545, "step_valid_time": 3.952734316000715, "step_total_time": 12119.821408033371, "elapsed_training_loop_seconds": 12119.821408033371, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0039918752614205, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08424704463921828, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1407, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1396, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9468371467025573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9455645161290323, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9462004034969738, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05443548387096774, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9394347240915208, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9381720430107527, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9388029589778076, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9468371467025573, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00534429527079, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09939995207265895, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1116, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1052, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1044, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9409660107334525, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.942652329749104, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9418084153983886, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05734767025089606, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9338103756708408, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9354838709677419, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9346463742166518, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9409660107334525, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0052610025659274, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09513686026105239, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 572, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 570, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.910828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9122807017543859, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9115537848605577, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08771929824561403, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9076433121019108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9090909090909091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9083665338645418, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.910828025477707, "step_test_time": 9.771784831013065, "step_plot_time": 7.30218300697743, "step_validation_cycle_time": 21.48941083799582} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_160000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_160000.json new file mode 100644 index 0000000000000000000000000000000000000000..1a536245f21b13bf9b38f531679eb8f8bf5e3d72 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_160000.json @@ -0,0 +1 @@ +{"epoch": 0.947311706996489, "train": {"Classification_binary": 2.433934450149536, "Classification": 0.017458004876971245, "Regression_pt": 0.16367757320404053, "Regression_eta": 0.013031451031565666, "Regression_sin_phi": 0.007010329514741898, "Regression_cos_phi": 0.007582052610814571, "Regression_energy": 0.16777151823043823, "Total": 6.413689613342285, "Total_unweighted": 2.8104653796181083}, "train_group": {}, "valid": {"Classification_binary": 2.579901674960522, "Classification": 0.017983053592925378, "Regression_pt": 0.14910246463532142, "Regression_eta": 0.013648929748129337, "Regression_sin_phi": 0.007790923118591309, "Regression_cos_phi": 0.00872488668624391, "Regression_energy": 0.15364540384170858, "Total": 6.26846638131649, "Total_unweighted": 2.930797336583442, "metrics/particle/class_1/efficiency": 0.9285033941268921, "metrics/particle/class_1/pid_accuracy": 0.9813264012336731, "metrics/particle/class_2/efficiency": 0.3948609232902527, "metrics/particle/class_2/pid_accuracy": 0.7319402694702148, "metrics/particle/class_3/efficiency": 0.767132580280304, "metrics/particle/class_3/pid_accuracy": 0.9804823398590088, "metrics/particle/class_4/efficiency": 0.8048290014266968, "metrics/particle/class_4/pid_accuracy": 0.3700000047683716, "metrics/particle/class_5/efficiency": 0.9715447425842285, "metrics/particle/class_5/pid_accuracy": 0.5983263850212097, "metrics/particle/count/bias_mean": 1.3480000495910645, "metrics/particle/count/mae": 3.3559999465942383, "metrics/particle/count/prediction_mean": 71.07599639892578, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.05913322791457176, "metrics/particle/event/energy_response_mean": 1.0200660228729248, "metrics/particle/event/met_abs_error": 5.930116653442383, "metrics/particle/event/scalar_pt_relative_abs_error": 0.049140818417072296, "metrics/particle/event/scalar_pt_response_mean": 1.0201987028121948, "metrics/particle/event/vector_pt_closure": 0.05356365442276001, "metrics/particle/matched/delta_eta_abs_mean": 0.014821039512753487, "metrics/particle/matched/delta_phi_abs_mean": 0.015310125425457954, "metrics/particle/matched/delta_r_mean": 0.02381850592792034, "metrics/particle/matched/energy_relative_abs_mean": 0.08816107362508774, "metrics/particle/matched/pid_accuracy": 0.9633274674415588, "metrics/particle/matched/pt_relative_abs_mean": 0.08360907435417175, "metrics/particle/matching/duplicate_fraction": 0.034648358821868896, "metrics/particle/matching/efficiency": 0.8160279989242554, "metrics/particle/matching/f1": 0.8082156777381897, "metrics/particle/matching/purity": 0.8005515336990356, "metrics/particle/matching/target_coverage_dr0p05": 0.7264800071716309, "metrics/particle/matching/target_coverage_dr0p10": 0.8485543727874756, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7266712784767151, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8660700917243958}, "step_train_time": 5.896226167678833, "step_valid_time": 4.402235812012805, "step_total_time": 12935.07180261612, "elapsed_training_loop_seconds": 12935.07180261612, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0035162757031006, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08316781899693093, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1496, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1407, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9542395693135935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9478609625668449, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9510395707578806, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05213903743315508, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9468371467025573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9405080213903744, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9436619718309859, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9542395693135935, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0011774986627038, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0937067854017084, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1047, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1038, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9364937388193202, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.935656836461126, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9360751005811354, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.064343163538874, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9284436493738819, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9276139410187667, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9280286097451944, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9364937388193202, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0064740481472214, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09835920774351714, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 559, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 556, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8901273885350318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8901273885350318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8901273885350318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10987261146496816, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8853503184713376, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8853503184713376, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8853503184713376, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8901273885350318, "step_test_time": 9.264800154982368, "step_plot_time": 7.382277650991455, "step_validation_cycle_time": 21.52169563699863} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_170000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_170000.json new file mode 100644 index 0000000000000000000000000000000000000000..ae51ff67d32f922589ca2294fadb2310249be2da --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_170000.json @@ -0,0 +1 @@ +{"epoch": 1.0065186886837696, "train": {"Classification_binary": 2.936152219772339, "Classification": 0.011880706995725632, "Regression_pt": 0.15002021193504333, "Regression_eta": 0.00847030058503151, "Regression_sin_phi": 0.004154957830905914, "Regression_cos_phi": 0.006944405380636454, "Regression_energy": 0.15033011138439178, "Total": 6.466993808746338, "Total_unweighted": 3.2679529138840735}, "train_group": {}, "valid": {"Classification_binary": 2.533518040433843, "Classification": 0.01758063093144843, "Regression_pt": 0.1454584649268617, "Regression_eta": 0.013341002007748218, "Regression_sin_phi": 0.0075849142480403815, "Regression_cos_phi": 0.00810556208833735, "Regression_energy": 0.15098921796108813, "Total": 6.140176975980718, "Total_unweighted": 2.876577832597367, "metrics/particle/class_1/efficiency": 0.9312807321548462, "metrics/particle/class_1/pid_accuracy": 0.9810412526130676, "metrics/particle/class_2/efficiency": 0.41819897294044495, "metrics/particle/class_2/pid_accuracy": 0.7683202028274536, "metrics/particle/class_3/efficiency": 0.7845478653907776, "metrics/particle/class_3/pid_accuracy": 0.9797047972679138, "metrics/particle/class_4/efficiency": 0.8430583477020264, "metrics/particle/class_4/pid_accuracy": 0.3818615674972534, "metrics/particle/class_5/efficiency": 0.9674796462059021, "metrics/particle/class_5/pid_accuracy": 0.6428571343421936, "metrics/particle/count/bias_mean": 1.027999997138977, "metrics/particle/count/mae": 3.2733333110809326, "metrics/particle/count/prediction_mean": 70.75599670410156, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.058663737028837204, "metrics/particle/event/energy_response_mean": 1.018028736114502, "metrics/particle/event/met_abs_error": 6.481718063354492, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04881646856665611, "metrics/particle/event/scalar_pt_response_mean": 1.0218677520751953, "metrics/particle/event/vector_pt_closure": 0.054296135902404785, "metrics/particle/matched/delta_eta_abs_mean": 0.01608278788626194, "metrics/particle/matched/delta_phi_abs_mean": 0.013914213515818119, "metrics/particle/matched/delta_r_mean": 0.02364523708820343, "metrics/particle/matched/energy_relative_abs_mean": 0.08877384662628174, "metrics/particle/matched/pid_accuracy": 0.9641131162643433, "metrics/particle/matched/pt_relative_abs_mean": 0.08398472517728806, "metrics/particle/matching/duplicate_fraction": 0.03403244912624359, "metrics/particle/matching/efficiency": 0.8269657492637634, "metrics/particle/matching/f1": 0.8209143877029419, "metrics/particle/matching/purity": 0.8149508833885193, "metrics/particle/matching/target_coverage_dr0p05": 0.7388327717781067, "metrics/particle/matching/target_coverage_dr0p10": 0.8603143692016602, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7314326167106628, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8711373805999756}, "step_train_time": 5.464946269989014, "step_valid_time": 4.43880552900373, "step_total_time": 13742.240288257599, "elapsed_training_loop_seconds": 13742.240288257599, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0047256633390143, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07703638249053912, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1402, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.949528936742934, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9533783783783784, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9514497639919083, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046621621621621624, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9434724091520862, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9472972972972973, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9453809844908968, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.949528936742934, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0072095174702551, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09226329349046636, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1114, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1059, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1052, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.947227191413238, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9506283662477558, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9489247311827957, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04937163375224417, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9409660107334525, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9443447037701975, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.942652329749104, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.947227191413238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0123879768413209, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10243694392587066, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 564, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 561, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8980891719745223, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9038461538461539, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9009584664536742, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09615384615384616, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.893312101910828, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8990384615384616, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8961661341853036, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8980891719745223, "step_test_time": 9.439136428001802, "step_plot_time": 7.307894005993148, "step_validation_cycle_time": 21.648543979012175} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_180000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_180000.json new file mode 100644 index 0000000000000000000000000000000000000000..988e2e56545699640526811ddcbb792252ca0cea --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_180000.json @@ -0,0 +1 @@ +{"epoch": 1.0657256703710503, "train": {"Classification_binary": 2.562946319580078, "Classification": 0.01850002259016037, "Regression_pt": 0.13570083677768707, "Regression_eta": 0.024614481255412102, "Regression_sin_phi": 0.006995968986302614, "Regression_cos_phi": 0.019444553181529045, "Regression_energy": 0.14313830435276031, "Total": 6.096354007720947, "Total_unweighted": 2.9113404867239296}, "train_group": {}, "valid": {"Classification_binary": 2.507194032060339, "Classification": 0.01732277996996616, "Regression_pt": 0.14661438921664624, "Regression_eta": 0.01375610016761942, "Regression_sin_phi": 0.007885203082510766, "Regression_cos_phi": 0.008386882695745914, "Regression_energy": 0.15117577289013154, "Total": 6.132959893409242, "Total_unweighted": 2.852335160082959, "metrics/particle/class_1/efficiency": 0.9392556548118591, "metrics/particle/class_1/pid_accuracy": 0.9846660494804382, "metrics/particle/class_2/efficiency": 0.3948609232902527, "metrics/particle/class_2/pid_accuracy": 0.7474626898765564, "metrics/particle/class_3/efficiency": 0.7534717321395874, "metrics/particle/class_3/pid_accuracy": 0.9818694591522217, "metrics/particle/class_4/efficiency": 0.818913459777832, "metrics/particle/class_4/pid_accuracy": 0.3390663266181946, "metrics/particle/class_5/efficiency": 0.9756097793579102, "metrics/particle/class_5/pid_accuracy": 0.5291666388511658, "metrics/particle/count/bias_mean": 0.15066666901111603, "metrics/particle/count/mae": 3.068000078201294, "metrics/particle/count/prediction_mean": 69.87866973876953, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.05092310532927513, "metrics/particle/event/energy_response_mean": 0.9997061491012573, "metrics/particle/event/met_abs_error": 5.356996059417725, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0427672304213047, "metrics/particle/event/scalar_pt_response_mean": 1.0069962739944458, "metrics/particle/event/vector_pt_closure": 0.048633869737386703, "metrics/particle/matched/delta_eta_abs_mean": 0.014820188283920288, "metrics/particle/matched/delta_phi_abs_mean": 0.01550713088363409, "metrics/particle/matched/delta_r_mean": 0.02396423928439617, "metrics/particle/matched/energy_relative_abs_mean": 0.08731550723314285, "metrics/particle/matched/pid_accuracy": 0.965535044670105, "metrics/particle/matched/pt_relative_abs_mean": 0.0831335112452507, "metrics/particle/matching/duplicate_fraction": 0.0311015285551548, "metrics/particle/matching/efficiency": 0.8155881762504578, "metrics/particle/matching/f1": 0.8147079944610596, "metrics/particle/matching/purity": 0.8138296604156494, "metrics/particle/matching/target_coverage_dr0p05": 0.723057210445404, "metrics/particle/matching/target_coverage_dr0p10": 0.8471776247024536, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7238221168518066, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8645976781845093}, "step_train_time": 5.083059072494507, "step_valid_time": 4.465136990998872, "step_total_time": 14532.419671058655, "elapsed_training_loop_seconds": 14532.419671058655, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0015898390411995, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07632192379869374, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1489, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1405, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9508748317631225, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.948959032907992, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9499159663865546, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05104096709200806, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9454912516823688, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9435862995298858, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9445378151260504, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9508748317631225, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0016746254589446, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08988808751912636, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1124, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1050, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1043, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9391771019677997, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9341637010676157, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.936663693131133, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06583629893238434, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9329159212880143, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9279359430604982, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9304192685102587, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9391771019677997, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0063902989300513, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08975560766339119, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 563, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 563, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8964968152866242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8964968152866242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8964968152866242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1035031847133758, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8964968152866242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8964968152866242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8964968152866242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8964968152866242, "step_test_time": 9.507251962000737, "step_plot_time": 7.262110838026274, "step_validation_cycle_time": 21.695942045014817} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_190000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_190000.json new file mode 100644 index 0000000000000000000000000000000000000000..5c235bf3c471c694c4ddfe2dc2707a631b807105 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_190000.json @@ -0,0 +1 @@ +{"epoch": 1.1249326520583307, "train": {"Classification_binary": 2.3106744289398193, "Classification": 0.01591760851442814, "Regression_pt": 0.12301437556743622, "Regression_eta": 0.0027922450099140406, "Regression_sin_phi": 0.003382232505828142, "Regression_cos_phi": 0.0032478980720043182, "Regression_energy": 0.1222945973277092, "Total": 5.2395477294921875, "Total_unweighted": 2.5813233859371394}, "train_group": {}, "valid": {"Classification_binary": 2.4730614195478724, "Classification": 0.01725772847520544, "Regression_pt": 0.1440941019261137, "Regression_eta": 0.013073921203613281, "Regression_sin_phi": 0.007549311252350503, "Regression_cos_phi": 0.008231207411339942, "Regression_energy": 0.14888438772647938, "Total": 6.038431370511968, "Total_unweighted": 2.812152077542975, "metrics/particle/class_1/efficiency": 0.9365973472595215, "metrics/particle/class_1/pid_accuracy": 0.9834364056587219, "metrics/particle/class_2/efficiency": 0.42197075486183167, "metrics/particle/class_2/pid_accuracy": 0.7603352069854736, "metrics/particle/class_3/efficiency": 0.7788935899734497, "metrics/particle/class_3/pid_accuracy": 0.9836227297782898, "metrics/particle/class_4/efficiency": 0.8289738297462463, "metrics/particle/class_4/pid_accuracy": 0.4150485396385193, "metrics/particle/class_5/efficiency": 0.9715447425842285, "metrics/particle/class_5/pid_accuracy": 0.6485355496406555, "metrics/particle/count/bias_mean": 0.5226666927337646, "metrics/particle/count/mae": 3.1040000915527344, "metrics/particle/count/prediction_mean": 70.25066375732422, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04872281476855278, "metrics/particle/event/energy_response_mean": 0.9857115745544434, "metrics/particle/event/met_abs_error": 5.06576681137085, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0413350835442543, "metrics/particle/event/scalar_pt_response_mean": 0.992642343044281, "metrics/particle/event/vector_pt_closure": 0.045859623700380325, "metrics/particle/matched/delta_eta_abs_mean": 0.013473397120833397, "metrics/particle/matched/delta_phi_abs_mean": 0.014967624098062515, "metrics/particle/matched/delta_r_mean": 0.022414663806557655, "metrics/particle/matched/energy_relative_abs_mean": 0.08667635172605515, "metrics/particle/matched/pid_accuracy": 0.9670180082321167, "metrics/particle/matched/pt_relative_abs_mean": 0.08287926763296127, "metrics/particle/matching/duplicate_fraction": 0.02917172759771347, "metrics/particle/matching/efficiency": 0.8273290395736694, "metrics/particle/matching/f1": 0.8242399096488953, "metrics/particle/matching/purity": 0.8211737275123596, "metrics/particle/matching/target_coverage_dr0p05": 0.7455063462257385, "metrics/particle/matching/target_coverage_dr0p10": 0.8566429615020752, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.734377384185791, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8731260299682617}, "step_train_time": 5.026301383972168, "step_valid_time": 4.458454289997462, "step_total_time": 15310.262139558792, "elapsed_training_loop_seconds": 15310.262139558792, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9997894585775364, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07402019111094849, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1412, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1406, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9502018842530283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9534098582039163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9518031681833502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04659014179608373, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.946164199192463, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.949358541525996, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9477586788001349, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9502018842530283, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9968223691528154, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08007962472935148, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1121, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1054, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1048, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9427549194991055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9402319357716324, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9414917373827602, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05976806422836753, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9373881932021467, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9348795718108831, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9361322018758375, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9427549194991055, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9956278449120277, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0926780950779843, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 572, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 570, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.910828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.910828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.910828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08917197452229299, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9076433121019108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9076433121019108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9076433121019108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.910828025477707, "step_test_time": 9.395016016002046, "step_plot_time": 7.318432744999882, "step_validation_cycle_time": 21.636310705012875} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_20000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_20000.json new file mode 100644 index 0000000000000000000000000000000000000000..c79d8b8dde327ede0272a907def09faa8803db73 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_20000.json @@ -0,0 +1 @@ +{"epoch": 0.11841396337456113, "train": {"Classification_binary": 3.991624355316162, "Classification": 0.035779230296611786, "Regression_pt": 0.23789361119270325, "Regression_eta": 0.035850752145051956, "Regression_sin_phi": 0.014545302838087082, "Regression_cos_phi": 0.015873640775680542, "Regression_energy": 0.24942897260189056, "Total": 10.09478759765625, "Total_unweighted": 4.580995865166187}, "train_group": {}, "valid": {"Classification_binary": 3.981923854097407, "Classification": 0.0317570224721381, "Regression_pt": 0.2510825015128927, "Regression_eta": 0.02293263851328099, "Regression_sin_phi": 0.011995406861000873, "Regression_cos_phi": 0.013978251751433027, "Regression_energy": 0.2598981451480947, "Total": 10.208237018991024, "Total_unweighted": 4.5735678203562475, "metrics/particle/class_1/efficiency": 0.8719251155853271, "metrics/particle/class_1/pid_accuracy": 0.9792045950889587, "metrics/particle/class_2/efficiency": 0.18481847643852234, "metrics/particle/class_2/pid_accuracy": 0.32397958636283875, "metrics/particle/class_3/efficiency": 0.671280562877655, "metrics/particle/class_3/pid_accuracy": 0.9731805920600891, "metrics/particle/class_4/efficiency": 0.8148893117904663, "metrics/particle/class_4/pid_accuracy": 0.0, "metrics/particle/class_5/efficiency": 0.9552845358848572, "metrics/particle/class_5/pid_accuracy": 0.07234042882919312, "metrics/particle/count/bias_mean": -0.35733333230018616, "metrics/particle/count/mae": 4.5226664543151855, "metrics/particle/count/prediction_mean": 69.37066650390625, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.09928470104932785, "metrics/particle/event/energy_response_mean": 0.998938798904419, "metrics/particle/event/met_abs_error": 11.840254783630371, "metrics/particle/event/scalar_pt_relative_abs_error": 0.08926833420991898, "metrics/particle/event/scalar_pt_response_mean": 1.0047047138214111, "metrics/particle/event/vector_pt_closure": 0.09827034175395966, "metrics/particle/matched/delta_eta_abs_mean": 0.024161528795957565, "metrics/particle/matched/delta_phi_abs_mean": 0.018396761268377304, "metrics/particle/matched/delta_r_mean": 0.03366805613040924, "metrics/particle/matched/energy_relative_abs_mean": 0.09714657068252563, "metrics/particle/matched/pid_accuracy": 0.9474895596504211, "metrics/particle/matched/pt_relative_abs_mean": 0.09331990033388138, "metrics/particle/matching/duplicate_fraction": 0.04115092009305954, "metrics/particle/matching/efficiency": 0.7312222719192505, "metrics/particle/matching/f1": 0.7331007122993469, "metrics/particle/matching/purity": 0.7349888682365417, "metrics/particle/matching/target_coverage_dr0p05": 0.5864884257316589, "metrics/particle/matching/target_coverage_dr0p10": 0.7645135521888733, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6590369939804077, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8006539940834045}, "step_train_time": 5.5626044273376465, "step_valid_time": 3.895395269006258, "step_total_time": 1609.4515850543976, "elapsed_training_loop_seconds": 1609.4515850543976, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9881036420054091, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.19557755315075198, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1335, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1256, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8983849259757739, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.898989898989899, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8986873106698081, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.10101010101010101, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.845222072678331, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8457912457912458, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.845506563446651, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8983849259757739, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9890242066460206, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.2075531810324569, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1116, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 972, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 943, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8694096601073346, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8709677419354839, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8701880035810206, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.12903225806451613, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8434704830053668, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8449820788530465, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8442256042972247, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8694096601073346, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0003325134377872, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.18557496622410852, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 633, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 489, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 479, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7786624203821656, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.7725118483412322, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.7755749405233942, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.22748815165876776, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7627388535031847, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7567140600315956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7597145122918318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7786624203821656, "step_test_time": 8.931686547992285, "step_plot_time": 7.134839952981565, "step_validation_cycle_time": 20.40454818800208} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_200000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_200000.json new file mode 100644 index 0000000000000000000000000000000000000000..9e4122d913e824d9d25d11892c135f36c3971f18 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_200000.json @@ -0,0 +1 @@ +{"epoch": 1.1841396337456114, "train": {"Classification_binary": 2.2660305500030518, "Classification": 0.016914937645196915, "Regression_pt": 0.10325801372528076, "Regression_eta": 0.02274957485496998, "Regression_sin_phi": 0.006115268915891647, "Regression_cos_phi": 0.016563333570957184, "Regression_energy": 0.11633903533220291, "Total": 5.09267520904541, "Total_unweighted": 2.547970714047551}, "train_group": {}, "valid": {"Classification_binary": 2.451465850180768, "Classification": 0.01716137186009833, "Regression_pt": 0.14022001307061377, "Regression_eta": 0.012795941626771968, "Regression_sin_phi": 0.007396329590614805, "Regression_cos_phi": 0.008281829509329288, "Regression_energy": 0.1457783212053015, "Total": 5.936067621758643, "Total_unweighted": 2.7830996570434974, "metrics/particle/class_1/efficiency": 0.936478316783905, "metrics/particle/class_1/pid_accuracy": 0.9846205711364746, "metrics/particle/class_2/efficiency": 0.41607731580734253, "metrics/particle/class_2/pid_accuracy": 0.7444759011268616, "metrics/particle/class_3/efficiency": 0.7918306589126587, "metrics/particle/class_3/pid_accuracy": 0.986918032169342, "metrics/particle/class_4/efficiency": 0.8410462737083435, "metrics/particle/class_4/pid_accuracy": 0.42583730816841125, "metrics/particle/class_5/efficiency": 0.9674796462059021, "metrics/particle/class_5/pid_accuracy": 0.6722689270973206, "metrics/particle/count/bias_mean": 0.5146666765213013, "metrics/particle/count/mae": 3.013333320617676, "metrics/particle/count/prediction_mean": 70.24266815185547, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.05146050080657005, "metrics/particle/event/energy_response_mean": 0.9724414944648743, "metrics/particle/event/met_abs_error": 5.040501594543457, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04150361940264702, "metrics/particle/event/scalar_pt_response_mean": 0.9798316955566406, "metrics/particle/event/vector_pt_closure": 0.04550311341881752, "metrics/particle/matched/delta_eta_abs_mean": 0.01387125812470913, "metrics/particle/matched/delta_phi_abs_mean": 0.013875511474907398, "metrics/particle/matched/delta_r_mean": 0.021876273676753044, "metrics/particle/matched/energy_relative_abs_mean": 0.08676540851593018, "metrics/particle/matched/pid_accuracy": 0.9687334895133972, "metrics/particle/matched/pt_relative_abs_mean": 0.08233924955129623, "metrics/particle/matching/duplicate_fraction": 0.028624577447772026, "metrics/particle/matching/efficiency": 0.8323581218719482, "metrics/particle/matching/f1": 0.8292975425720215, "metrics/particle/matching/purity": 0.826259434223175, "metrics/particle/matching/target_coverage_dr0p05": 0.7600581049919128, "metrics/particle/matching/target_coverage_dr0p10": 0.8605055809020996, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7389284372329712, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8764150142669678}, "step_train_time": 5.106242656707764, "step_valid_time": 3.9541922690114006, "step_total_time": 16091.040839195251, "elapsed_training_loop_seconds": 16091.040839195251, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9961466760858416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07690111330459237, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1409, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1401, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9481830417227456, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.952027027027027, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9501011463250169, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047972972972972976, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9427994616419919, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9466216216216217, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9447066756574511, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9481830417227456, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9866066467027178, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.09163090168969479, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1116, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1063, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1056, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9508050089445438, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9525089605734767, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9516562220232766, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.047491039426523295, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9445438282647585, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.946236559139785, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.945389435989257, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9508050089445438, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9892720494946237, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08879802440365703, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 623, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 571, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 570, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9092356687898089, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9165329052969502, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9128697042366107, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08346709470304976, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9076433121019108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9149277688603531, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9112709832134293, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9092356687898089, "step_test_time": 10.038056761986809, "step_plot_time": 7.229716320987791, "step_validation_cycle_time": 21.68478621097165} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_210000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_210000.json new file mode 100644 index 0000000000000000000000000000000000000000..22688888aac35dd16e01f099f0928b8804b4a5ae --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_210000.json @@ -0,0 +1 @@ +{"epoch": 1.2433466154328918, "train": {"Classification_binary": 2.46769642829895, "Classification": 0.014292441308498383, "Regression_pt": 0.09841229766607285, "Regression_eta": 0.014058525674045086, "Regression_sin_phi": 0.010570000857114792, "Regression_cos_phi": 0.00710332253947854, "Regression_energy": 0.10774464905261993, "Total": 5.066508769989014, "Total_unweighted": 2.7198776653967798}, "train_group": {}, "valid": {"Classification_binary": 2.4143204384661736, "Classification": 0.01693808651985006, "Regression_pt": 0.1350856030240972, "Regression_eta": 0.013116009691928296, "Regression_sin_phi": 0.0073836794558991775, "Regression_cos_phi": 0.0080173725777484, "Regression_energy": 0.1402109531646079, "Total": 5.780080592378657, "Total_unweighted": 2.735072142900304, "metrics/particle/class_1/efficiency": 0.9385811686515808, "metrics/particle/class_1/pid_accuracy": 0.9849086999893188, "metrics/particle/class_2/efficiency": 0.4177275002002716, "metrics/particle/class_2/pid_accuracy": 0.7562077045440674, "metrics/particle/class_3/efficiency": 0.7908807396888733, "metrics/particle/class_3/pid_accuracy": 0.986616313457489, "metrics/particle/class_4/efficiency": 0.8289738297462463, "metrics/particle/class_4/pid_accuracy": 0.4126213490962982, "metrics/particle/class_5/efficiency": 0.9715447425842285, "metrics/particle/class_5/pid_accuracy": 0.6652719378471375, "metrics/particle/count/bias_mean": 0.3186666667461395, "metrics/particle/count/mae": 2.990666627883911, "metrics/particle/count/prediction_mean": 70.04666900634766, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.048492684960365295, "metrics/particle/event/energy_response_mean": 1.0063656568527222, "metrics/particle/event/met_abs_error": 4.781017780303955, "metrics/particle/event/scalar_pt_relative_abs_error": 0.040523432195186615, "metrics/particle/event/scalar_pt_response_mean": 1.009671926498413, "metrics/particle/event/vector_pt_closure": 0.044687338173389435, "metrics/particle/matched/delta_eta_abs_mean": 0.014624103903770447, "metrics/particle/matched/delta_phi_abs_mean": 0.013252208940684795, "metrics/particle/matched/delta_r_mean": 0.021973002701997757, "metrics/particle/matched/energy_relative_abs_mean": 0.08647986501455307, "metrics/particle/matched/pid_accuracy": 0.9691251516342163, "metrics/particle/matched/pt_relative_abs_mean": 0.08188298344612122, "metrics/particle/matching/duplicate_fraction": 0.029294755309820175, "metrics/particle/matching/efficiency": 0.8330082893371582, "metrics/particle/matching/f1": 0.8311091065406799, "metrics/particle/matching/purity": 0.8292186260223389, "metrics/particle/matching/target_coverage_dr0p05": 0.7580311894416809, "metrics/particle/matching/target_coverage_dr0p10": 0.8616529107093811, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7412803769111633, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8770269155502319}, "step_train_time": 5.3180272579193115, "step_valid_time": 3.969976535998285, "step_total_time": 16892.852580308914, "elapsed_training_loop_seconds": 16892.852580308914, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0004046056672373, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0678398725116709, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9562584118438762, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9569023569023569, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9565802760013463, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0430976430976431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9508748317631225, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9515151515151515, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9511948838774823, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9562584118438762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0001017134291688, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08542180054576654, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1116, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1065, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9543828264758497, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9560931899641577, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.955237242614145, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04390681003584229, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9525939177101967, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9543010752688172, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534467323187108, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9543828264758497, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.007126396117855, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08700453553221676, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 622, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 573, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 571, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9124203821656051, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9212218649517685, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9168, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07877813504823152, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9092356687898089, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9180064308681672, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9136, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9124203821656051, "step_test_time": 9.245446679007728, "step_plot_time": 7.78931295798975, "step_validation_cycle_time": 21.468747043982148} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_220000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_220000.json new file mode 100644 index 0000000000000000000000000000000000000000..118bfa362aa293dd159f27ac810dd05aa4de7d61 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_220000.json @@ -0,0 +1 @@ +{"epoch": 1.3025535971201725, "train": {"Classification_binary": 2.5803027153015137, "Classification": 0.015109125524759293, "Regression_pt": 0.11864233762025833, "Regression_eta": 0.017983686178922653, "Regression_sin_phi": 0.010117911733686924, "Regression_cos_phi": 0.004840067587792873, "Regression_energy": 0.13289840519428253, "Total": 5.676135540008545, "Total_unweighted": 2.8798942491412163}, "train_group": {}, "valid": {"Classification_binary": 2.4086018014461437, "Classification": 0.01667979042580787, "Regression_pt": 0.13501304261227873, "Regression_eta": 0.012743296775412052, "Regression_sin_phi": 0.0071632411885768806, "Regression_cos_phi": 0.007995218672650926, "Regression_energy": 0.1404334230625883, "Total": 5.76869493849734, "Total_unweighted": 2.7286298141834586, "metrics/particle/class_1/efficiency": 0.9386605024337769, "metrics/particle/class_1/pid_accuracy": 0.9826274514198303, "metrics/particle/class_2/efficiency": 0.4429514408111572, "metrics/particle/class_2/pid_accuracy": 0.766365110874176, "metrics/particle/class_3/efficiency": 0.7856787443161011, "metrics/particle/class_3/pid_accuracy": 0.9858943819999695, "metrics/particle/class_4/efficiency": 0.8350100517272949, "metrics/particle/class_4/pid_accuracy": 0.45301204919815063, "metrics/particle/class_5/efficiency": 0.9593495726585388, "metrics/particle/class_5/pid_accuracy": 0.6228813529014587, "metrics/particle/count/bias_mean": 0.8946666717529297, "metrics/particle/count/mae": 3.118666648864746, "metrics/particle/count/prediction_mean": 70.62266540527344, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.05417990684509277, "metrics/particle/event/energy_response_mean": 1.0260303020477295, "metrics/particle/event/met_abs_error": 5.594020843505859, "metrics/particle/event/scalar_pt_relative_abs_error": 0.05056461691856384, "metrics/particle/event/scalar_pt_response_mean": 1.03455650806427, "metrics/particle/event/vector_pt_closure": 0.04768247529864311, "metrics/particle/matched/delta_eta_abs_mean": 0.013357482850551605, "metrics/particle/matched/delta_phi_abs_mean": 0.01402697991579771, "metrics/particle/matched/delta_r_mean": 0.02162798121571541, "metrics/particle/matched/energy_relative_abs_mean": 0.08726424723863602, "metrics/particle/matched/pid_accuracy": 0.9676056504249573, "metrics/particle/matched/pt_relative_abs_mean": 0.0835408866405487, "metrics/particle/matching/duplicate_fraction": 0.0338701456785202, "metrics/particle/matching/efficiency": 0.8328935503959656, "metrics/particle/matching/f1": 0.8275842666625977, "metrics/particle/matching/purity": 0.8223422169685364, "metrics/particle/matching/target_coverage_dr0p05": 0.7614540457725525, "metrics/particle/matching/target_coverage_dr0p10": 0.8661274313926697, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.736021876335144, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8751912117004395}, "step_train_time": 5.21453595161438, "step_valid_time": 3.9639544740202837, "step_total_time": 17706.483971834183, "elapsed_training_loop_seconds": 17706.483971834183, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.013926679306578, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07294172542706845, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9562584118438762, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9575471698113207, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9569023569023569, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04245283018867924, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9508748317631225, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9521563342318059, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9515151515151515, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9562584118438762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0160113905524653, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08512080961257584, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1120, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1064, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1059, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9516994633273703, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.95, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9508489722966935, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.947227191413238, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9455357142857143, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9463806970509383, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9516994633273703, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0224588429629646, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.10024822850196446, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 568, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 567, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9044585987261147, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9030206677265501, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.903739061256961, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09697933227344992, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9028662420382165, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9014308426073132, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9021479713603818, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9044585987261147, "step_test_time": 9.323863255995093, "step_plot_time": 7.314617382013239, "step_validation_cycle_time": 21.188180922006723} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_230000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_230000.json new file mode 100644 index 0000000000000000000000000000000000000000..6a6b2955b787475748ae34dac1a73150d0a73147 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_230000.json @@ -0,0 +1 @@ +{"epoch": 1.361760578807453, "train": {"Classification_binary": 2.3967785835266113, "Classification": 0.017136521637439728, "Regression_pt": 0.0895022600889206, "Regression_eta": 0.01015546079725027, "Regression_sin_phi": 0.004836671054363251, "Regression_cos_phi": 0.0061566028743982315, "Regression_energy": 0.09354739636182785, "Total": 4.715967655181885, "Total_unweighted": 2.6181134963408113}, "train_group": {}, "valid": {"Classification_binary": 2.3564686876662235, "Classification": 0.01641360749589636, "Regression_pt": 0.13191368224773, "Regression_eta": 0.0130731260522883, "Regression_sin_phi": 0.00714963103862519, "Regression_cos_phi": 0.00787263347747478, "Regression_energy": 0.13713512014835438, "Total": 5.644566962059508, "Total_unweighted": 2.6700264881265925, "metrics/particle/class_1/efficiency": 0.9421917200088501, "metrics/particle/class_1/pid_accuracy": 0.9847138524055481, "metrics/particle/class_2/efficiency": 0.43517208099365234, "metrics/particle/class_2/pid_accuracy": 0.7692307829856873, "metrics/particle/class_3/efficiency": 0.7811552882194519, "metrics/particle/class_3/pid_accuracy": 0.98558109998703, "metrics/particle/class_4/efficiency": 0.8470824956893921, "metrics/particle/class_4/pid_accuracy": 0.40142518281936646, "metrics/particle/class_5/efficiency": 0.9756097793579102, "metrics/particle/class_5/pid_accuracy": 0.6208333373069763, "metrics/particle/count/bias_mean": -0.20666666328907013, "metrics/particle/count/mae": 2.859999895095825, "metrics/particle/count/prediction_mean": 69.52133178710938, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04997687786817551, "metrics/particle/event/energy_response_mean": 1.0102465152740479, "metrics/particle/event/met_abs_error": 4.583860397338867, "metrics/particle/event/scalar_pt_relative_abs_error": 0.04017361253499985, "metrics/particle/event/scalar_pt_response_mean": 1.0079631805419922, "metrics/particle/event/vector_pt_closure": 0.04608780890703201, "metrics/particle/matched/delta_eta_abs_mean": 0.014565009623765945, "metrics/particle/matched/delta_phi_abs_mean": 0.013202985748648643, "metrics/particle/matched/delta_r_mean": 0.021897967904806137, "metrics/particle/matched/energy_relative_abs_mean": 0.08789930492639542, "metrics/particle/matched/pid_accuracy": 0.9682696461677551, "metrics/particle/matched/pt_relative_abs_mean": 0.08204188197851181, "metrics/particle/matching/duplicate_fraction": 0.029957231134176254, "metrics/particle/matching/efficiency": 0.8322433829307556, "metrics/particle/matching/f1": 0.833478569984436, "metrics/particle/matching/purity": 0.8347173929214478, "metrics/particle/matching/target_coverage_dr0p05": 0.7528874278068542, "metrics/particle/matching/target_coverage_dr0p10": 0.8632974028587341, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7391005158424377, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.873030424118042}, "step_train_time": 6.206357955932617, "step_valid_time": 3.9615935610199813, "step_total_time": 18526.19540834427, "elapsed_training_loop_seconds": 18526.19540834427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9990360970792955, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07115263527655313, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1412, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9528936742934051, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.954177897574124, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9535353535353536, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04582210242587601, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9502018842530283, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9514824797843666, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9508417508417508, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9528936742934051, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9997053768953762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08440085108352902, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1116, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1059, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1055, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.947227191413238, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9489247311827957, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9480752014324082, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.051075268817204304, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.943649373881932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9453405017921147, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9444941808415398, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.947227191413238, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0050221014061393, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08654883854128226, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 567, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 567, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9028662420382165, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9072, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9050279329608939, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0928, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9028662420382165, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9072, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9050279329608939, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9028662420382165, "step_test_time": 9.26481887200498, "step_plot_time": 7.2957730739726685, "step_validation_cycle_time": 21.07136671798071} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_240000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_240000.json new file mode 100644 index 0000000000000000000000000000000000000000..8a7ccd84b05b2584bc16aa849d82ba823a8bcea7 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_240000.json @@ -0,0 +1 @@ +{"epoch": 1.4209675604947336, "train": {"Classification_binary": 2.3251655101776123, "Classification": 0.0165935680270195, "Regression_pt": 0.1487511545419693, "Regression_eta": 0.004994185175746679, "Regression_sin_phi": 0.004915856756269932, "Regression_cos_phi": 0.006189530715346336, "Regression_energy": 0.1460866630077362, "Total": 5.8450493812561035, "Total_unweighted": 2.6526964684017003}, "train_group": {}, "valid": {"Classification_binary": 2.341290737720246, "Classification": 0.016410851732213447, "Regression_pt": 0.13317371936554603, "Regression_eta": 0.012947090128634839, "Regression_sin_phi": 0.007084845862490066, "Regression_cos_phi": 0.007846169015194507, "Regression_energy": 0.13814627870600274, "Total": 5.65325927734375, "Total_unweighted": 2.6568996925303274, "metrics/particle/class_1/efficiency": 0.9422314167022705, "metrics/particle/class_1/pid_accuracy": 0.9841249585151672, "metrics/particle/class_2/efficiency": 0.4417727589607239, "metrics/particle/class_2/pid_accuracy": 0.7748132348060608, "metrics/particle/class_3/efficiency": 0.801284670829773, "metrics/particle/class_3/pid_accuracy": 0.9863384962081909, "metrics/particle/class_4/efficiency": 0.8370221257209778, "metrics/particle/class_4/pid_accuracy": 0.4639423191547394, "metrics/particle/class_5/efficiency": 0.9837398529052734, "metrics/particle/class_5/pid_accuracy": 0.7272727489471436, "metrics/particle/count/bias_mean": 1.1119999885559082, "metrics/particle/count/mae": 3.1066665649414062, "metrics/particle/count/prediction_mean": 70.83999633789062, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.0455649271607399, "metrics/particle/event/energy_response_mean": 0.9990081191062927, "metrics/particle/event/met_abs_error": 4.281099796295166, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036873769015073776, "metrics/particle/event/scalar_pt_response_mean": 0.9958850741386414, "metrics/particle/event/vector_pt_closure": 0.04172854498028755, "metrics/particle/matched/delta_eta_abs_mean": 0.013912709429860115, "metrics/particle/matched/delta_phi_abs_mean": 0.013264178298413754, "metrics/particle/matched/delta_r_mean": 0.02146040089428425, "metrics/particle/matched/energy_relative_abs_mean": 0.08768775314092636, "metrics/particle/matched/pid_accuracy": 0.9697685837745667, "metrics/particle/matched/pt_relative_abs_mean": 0.08278551697731018, "metrics/particle/matching/duplicate_fraction": 0.028119705617427826, "metrics/particle/matching/efficiency": 0.8412498235702515, "metrics/particle/matching/f1": 0.8345949053764343, "metrics/particle/matching/purity": 0.8280444145202637, "metrics/particle/matching/target_coverage_dr0p05": 0.7768663167953491, "metrics/particle/matching/target_coverage_dr0p10": 0.8720360994338989, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7455828189849854, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8818073868751526}, "step_train_time": 4.980132579803467, "step_valid_time": 3.948619870992843, "step_total_time": 19337.86821627617, "elapsed_training_loop_seconds": 19337.86821627617, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9943017086951799, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06877436091676581, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9562584118438762, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9549731182795699, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9556153328850033, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04502688172043011, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9528936742934051, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9516129032258065, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9522528581035642, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9562584118438762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9947010255460538, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08080124392218191, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1061, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9543828264758497, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.95013357079252, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9522534582775547, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04986642920747997, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9490161001788909, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9447907390917186, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9468987059348505, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9543828264758497, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9939640468894237, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08168908932131369, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 570, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 568, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9076433121019108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9090909090909091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9083665338645418, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09090909090909091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9044585987261147, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9059011164274322, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9051792828685259, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9076433121019108, "step_test_time": 9.359623035998084, "step_plot_time": 7.401415770989843, "step_validation_cycle_time": 21.22526609399938} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_250000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_250000.json new file mode 100644 index 0000000000000000000000000000000000000000..a3f95f31f406b45bce42f4249ecc6175c78c6cda --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_250000.json @@ -0,0 +1 @@ +{"epoch": 1.480174542182014, "train": {"Classification_binary": 2.26005482673645, "Classification": 0.014452625997364521, "Regression_pt": 0.1295889914035797, "Regression_eta": 0.01168775837868452, "Regression_sin_phi": 0.004652555100619793, "Regression_cos_phi": 0.0065898895263671875, "Regression_energy": 0.12661680579185486, "Total": 5.363192558288574, "Total_unweighted": 2.553643452934921}, "train_group": {}, "valid": {"Classification_binary": 2.324098140635389, "Classification": 0.016170978546142578, "Regression_pt": 0.13004020934409283, "Regression_eta": 0.012637328594288927, "Regression_sin_phi": 0.007166989940278074, "Regression_cos_phi": 0.0078300009382532, "Regression_energy": 0.13557784100796313, "Total": 5.56927490234375, "Total_unweighted": 2.633521489006408, "metrics/particle/class_1/efficiency": 0.9381844401359558, "metrics/particle/class_1/pid_accuracy": 0.9849869012832642, "metrics/particle/class_2/efficiency": 0.43352192640304565, "metrics/particle/class_2/pid_accuracy": 0.7759652137756348, "metrics/particle/class_3/efficiency": 0.7887547016143799, "metrics/particle/class_3/pid_accuracy": 0.9862934947013855, "metrics/particle/class_4/efficiency": 0.8350100517272949, "metrics/particle/class_4/pid_accuracy": 0.4457831382751465, "metrics/particle/class_5/efficiency": 0.9796748161315918, "metrics/particle/class_5/pid_accuracy": 0.6390041708946228, "metrics/particle/count/bias_mean": -0.15466666221618652, "metrics/particle/count/mae": 2.93066668510437, "metrics/particle/count/prediction_mean": 69.57333374023438, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.047725118696689606, "metrics/particle/event/energy_response_mean": 1.0055994987487793, "metrics/particle/event/met_abs_error": 4.779150485992432, "metrics/particle/event/scalar_pt_relative_abs_error": 0.038905851542949677, "metrics/particle/event/scalar_pt_response_mean": 1.010154128074646, "metrics/particle/event/vector_pt_closure": 0.043844789266586304, "metrics/particle/matched/delta_eta_abs_mean": 0.01604813151061535, "metrics/particle/matched/delta_phi_abs_mean": 0.013524282723665237, "metrics/particle/matched/delta_r_mean": 0.023358328267931938, "metrics/particle/matched/energy_relative_abs_mean": 0.09036365151405334, "metrics/particle/matched/pid_accuracy": 0.9696406722068787, "metrics/particle/matched/pt_relative_abs_mean": 0.0832604244351387, "metrics/particle/matching/duplicate_fraction": 0.029743196442723274, "metrics/particle/matching/efficiency": 0.8332951068878174, "metrics/particle/matching/f1": 0.834220290184021, "metrics/particle/matching/purity": 0.8351475596427917, "metrics/particle/matching/target_coverage_dr0p05": 0.7434602975845337, "metrics/particle/matching/target_coverage_dr0p10": 0.8619397282600403, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7431160807609558, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8758413791656494}, "step_train_time": 5.327682018280029, "step_valid_time": 3.970074826007476, "step_total_time": 20137.572376012802, "elapsed_training_loop_seconds": 20137.572376012802, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.009703228233084, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06738604649140156, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1479, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9542395693135935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9587559161595672, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9564924114671164, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.041244083840432724, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.949528936742934, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9540229885057471, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.95177065767285, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9542395693135935, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.006851206584218, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07688987862778429, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1065, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1061, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9525939177101967, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9534467323187108, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9530201342281879, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.046553267681289166, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9490161001788909, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9498657117278424, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9494407158836689, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9525939177101967, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0094908798538433, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0795871057389641, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 623, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 566, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 565, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9012738853503185, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9085072231139647, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9048760991207034, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09149277688603531, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8996815286624203, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9069020866773676, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.903277378097522, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9012738853503185, "step_test_time": 9.33562051798799, "step_plot_time": 7.287099314999068, "step_validation_cycle_time": 21.150148462998914} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_260000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_260000.json new file mode 100644 index 0000000000000000000000000000000000000000..45ca746a4ec82bedc12f2283e68582f6175de872 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_260000.json @@ -0,0 +1 @@ +{"epoch": 1.5393815238692947, "train": {"Classification_binary": 1.9196544885635376, "Classification": 0.019812973216176033, "Regression_pt": 0.0971883237361908, "Regression_eta": 0.0067923530004918575, "Regression_sin_phi": 0.004621028434485197, "Regression_cos_phi": 0.003568591782823205, "Regression_energy": 0.09876682609319687, "Total": 4.3845720291137695, "Total_unweighted": 2.1504045848269016}, "train_group": {}, "valid": {"Classification_binary": 2.3189885565575135, "Classification": 0.016144397410940616, "Regression_pt": 0.12794612316375084, "Regression_eta": 0.012593818471786823, "Regression_sin_phi": 0.007230336995835, "Regression_cos_phi": 0.007869438922151606, "Regression_energy": 0.13276659174168365, "Total": 5.512144048163232, "Total_unweighted": 2.623539263263662, "metrics/particle/class_1/efficiency": 0.9440168142318726, "metrics/particle/class_1/pid_accuracy": 0.9859622716903687, "metrics/particle/class_2/efficiency": 0.4323432445526123, "metrics/particle/class_2/pid_accuracy": 0.7797164916992188, "metrics/particle/class_3/efficiency": 0.793097198009491, "metrics/particle/class_3/pid_accuracy": 0.9853989481925964, "metrics/particle/class_4/efficiency": 0.8732394576072693, "metrics/particle/class_4/pid_accuracy": 0.41705068945884705, "metrics/particle/class_5/efficiency": 0.9837398529052734, "metrics/particle/class_5/pid_accuracy": 0.6074380278587341, "metrics/particle/count/bias_mean": 0.47600001096725464, "metrics/particle/count/mae": 2.9213333129882812, "metrics/particle/count/prediction_mean": 70.2040023803711, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.046167679131031036, "metrics/particle/event/energy_response_mean": 0.994827151298523, "metrics/particle/event/met_abs_error": 4.831456184387207, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03730276599526405, "metrics/particle/event/scalar_pt_response_mean": 1.0017956495285034, "metrics/particle/event/vector_pt_closure": 0.04513193294405937, "metrics/particle/matched/delta_eta_abs_mean": 0.013896588236093521, "metrics/particle/matched/delta_phi_abs_mean": 0.01275443471968174, "metrics/particle/matched/delta_r_mean": 0.021061331033706665, "metrics/particle/matched/energy_relative_abs_mean": 0.08460228145122528, "metrics/particle/matched/pid_accuracy": 0.9693859219551086, "metrics/particle/matched/pt_relative_abs_mean": 0.08042997866868973, "metrics/particle/matching/duplicate_fraction": 0.02977987937629223, "metrics/particle/matching/efficiency": 0.8382285237312317, "metrics/particle/matching/f1": 0.8353771567344666, "metrics/particle/matching/purity": 0.8325451612472534, "metrics/particle/matching/target_coverage_dr0p05": 0.7698103189468384, "metrics/particle/matching/target_coverage_dr0p10": 0.8681926131248474, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7477818727493286, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8798378705978394}, "step_train_time": 5.296939849853516, "step_valid_time": 4.004739066003822, "step_total_time": 20944.130472898483, "elapsed_training_loop_seconds": 20944.130472898483, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018362295644272, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.061772585954353976, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1411, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1406, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.949528936742934, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9501683501683502, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9498485358465163, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04983164983164983, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.946164199192463, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9468013468013468, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9464826657691013, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.949528936742934, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998457068043932, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07655138358361879, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1051, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1048, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9400715563506261, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9392314566577301, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9396513187304426, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06076854334226988, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9373881932021467, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.936550491510277, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9369691551184622, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9400715563506261, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0060034999365288, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07515903816090197, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 570, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 569, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9076433121019108, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9105431309904153, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9090909090909091, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08945686900958466, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9060509554140127, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9089456869009584, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9074960127591707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9076433121019108, "step_test_time": 9.292806932004169, "step_plot_time": 7.371390147018246, "step_validation_cycle_time": 21.23759995700675} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_270000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_270000.json new file mode 100644 index 0000000000000000000000000000000000000000..b82f2c486a7cd987849935b5a265d69800960ea5 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_270000.json @@ -0,0 +1 @@ +{"epoch": 1.5985885055565752, "train": {"Classification_binary": 2.2439053058624268, "Classification": 0.01745661348104477, "Regression_pt": 0.10968078672885895, "Regression_eta": 0.018295535817742348, "Regression_sin_phi": 0.010289115831255913, "Regression_cos_phi": 0.010159178636968136, "Regression_energy": 0.11677026748657227, "Total": 5.140397071838379, "Total_unweighted": 2.526556803844869}, "train_group": {}, "valid": {"Classification_binary": 2.2895080891061337, "Classification": 0.015793692558369737, "Regression_pt": 0.12683912033730366, "Regression_eta": 0.012707518770339642, "Regression_sin_phi": 0.00702370544697376, "Regression_cos_phi": 0.007741512136256441, "Regression_energy": 0.13259097363086456, "Total": 5.460872568982713, "Total_unweighted": 2.5922046119862414, "metrics/particle/class_1/efficiency": 0.9461593627929688, "metrics/particle/class_1/pid_accuracy": 0.9854488968849182, "metrics/particle/class_2/efficiency": 0.4460160434246063, "metrics/particle/class_2/pid_accuracy": 0.7838266491889954, "metrics/particle/class_3/efficiency": 0.7913330793380737, "metrics/particle/class_3/pid_accuracy": 0.9851949214935303, "metrics/particle/class_4/efficiency": 0.8410462737083435, "metrics/particle/class_4/pid_accuracy": 0.4354066848754883, "metrics/particle/class_5/efficiency": 0.9756097793579102, "metrics/particle/class_5/pid_accuracy": 0.6333333253860474, "metrics/particle/count/bias_mean": 0.2893333435058594, "metrics/particle/count/mae": 2.8386666774749756, "metrics/particle/count/prediction_mean": 70.017333984375, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.044420406222343445, "metrics/particle/event/energy_response_mean": 0.9956020712852478, "metrics/particle/event/met_abs_error": 4.544923305511475, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03658362850546837, "metrics/particle/event/scalar_pt_response_mean": 0.9968854784965515, "metrics/particle/event/vector_pt_closure": 0.040945474058389664, "metrics/particle/matched/delta_eta_abs_mean": 0.012983471155166626, "metrics/particle/matched/delta_phi_abs_mean": 0.0142106544226408, "metrics/particle/matched/delta_r_mean": 0.021481690928339958, "metrics/particle/matched/energy_relative_abs_mean": 0.08667094260454178, "metrics/particle/matched/pid_accuracy": 0.9694926142692566, "metrics/particle/matched/pt_relative_abs_mean": 0.08056165277957916, "metrics/particle/matching/duplicate_fraction": 0.02873574197292328, "metrics/particle/matching/efficiency": 0.8392802476882935, "metrics/particle/matching/f1": 0.8375425934791565, "metrics/particle/matching/purity": 0.8358120918273926, "metrics/particle/matching/target_coverage_dr0p05": 0.7640545964241028, "metrics/particle/matching/target_coverage_dr0p10": 0.8662995100021362, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7504971623420715, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8832415342330933}, "step_train_time": 5.046769142150879, "step_valid_time": 4.002730479987804, "step_total_time": 21757.89730477333, "elapsed_training_loop_seconds": 21757.89730477333, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9962869657123662, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06689985464840165, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1413, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9562584118438762, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9569023569023569, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9565802760013463, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0430976430976431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9508748317631225, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9515151515151515, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9511948838774823, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9562584118438762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9934445770815198, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07621741524655734, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1071, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1066, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9579606440071556, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9579606440071556, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9579606440071556, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04203935599284436, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9579606440071556, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998969394499217, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08146727239442542, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 623, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 580, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 579, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9235668789808917, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9309791332263242, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9272581934452439, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06902086677367576, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9293739967897271, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9256594724220624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9235668789808917, "step_test_time": 9.49751342899981, "step_plot_time": 7.314119994989596, "step_validation_cycle_time": 21.386166689015226} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_280000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_280000.json new file mode 100644 index 0000000000000000000000000000000000000000..597fcd0e9ed3d8c3b9d93407334dfb405516ac8a --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_280000.json @@ -0,0 +1 @@ +{"epoch": 1.6577954872438558, "train": {"Classification_binary": 2.2579009532928467, "Classification": 0.022533314302563667, "Regression_pt": 0.12002324312925339, "Regression_eta": 0.007086343131959438, "Regression_sin_phi": 0.01182840671390295, "Regression_cos_phi": 0.010466312989592552, "Regression_energy": 0.11847627907991409, "Total": 5.326920032501221, "Total_unweighted": 2.5483148526400328}, "train_group": {}, "valid": {"Classification_binary": 2.278829371675532, "Classification": 0.015922386595543396, "Regression_pt": 0.12765932083129883, "Regression_eta": 0.012318460231131695, "Regression_sin_phi": 0.007033197803700224, "Regression_cos_phi": 0.007653923110759004, "Regression_energy": 0.13257928604775288, "Total": 5.4598025058178195, "Total_unweighted": 2.581995946295718, "metrics/particle/class_1/efficiency": 0.9431836009025574, "metrics/particle/class_1/pid_accuracy": 0.9856974482536316, "metrics/particle/class_2/efficiency": 0.4441301226615906, "metrics/particle/class_2/pid_accuracy": 0.7882165312767029, "metrics/particle/class_3/efficiency": 0.7880309224128723, "metrics/particle/class_3/pid_accuracy": 0.9855921268463135, "metrics/particle/class_4/efficiency": 0.8470824956893921, "metrics/particle/class_4/pid_accuracy": 0.4346793293952942, "metrics/particle/class_5/efficiency": 0.9796748161315918, "metrics/particle/class_5/pid_accuracy": 0.6597510576248169, "metrics/particle/count/bias_mean": -0.36800000071525574, "metrics/particle/count/mae": 2.8506667613983154, "metrics/particle/count/prediction_mean": 69.36000061035156, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.0459090918302536, "metrics/particle/event/energy_response_mean": 0.9929894208908081, "metrics/particle/event/met_abs_error": 4.670773983001709, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03686267137527466, "metrics/particle/event/scalar_pt_response_mean": 0.9948350191116333, "metrics/particle/event/vector_pt_closure": 0.041698258370161057, "metrics/particle/matched/delta_eta_abs_mean": 0.013545800000429153, "metrics/particle/matched/delta_phi_abs_mean": 0.013995221816003323, "metrics/particle/matched/delta_r_mean": 0.021699383854866028, "metrics/particle/matched/energy_relative_abs_mean": 0.08773783594369888, "metrics/particle/matched/pid_accuracy": 0.9700496196746826, "metrics/particle/matched/pt_relative_abs_mean": 0.08234721422195435, "metrics/particle/matching/duplicate_fraction": 0.02691272646188736, "metrics/particle/matching/efficiency": 0.8363737463951111, "metrics/particle/matching/f1": 0.8385866284370422, "metrics/particle/matching/purity": 0.8408112525939941, "metrics/particle/matching/target_coverage_dr0p05": 0.7660241723060608, "metrics/particle/matching/target_coverage_dr0p10": 0.8676954507827759, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7372839450836182, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8747513890266418}, "step_train_time": 5.221513032913208, "step_valid_time": 4.013225730013801, "step_total_time": 22556.370512247086, "elapsed_training_loop_seconds": 22556.370512247086, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009211227520238, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06759192423204641, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1491, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1425, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9589502018842531, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9557344064386318, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9573396036278132, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04426559356136821, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9542395693135935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9510395707578806, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9526368827678872, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9589502018842531, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9993620336871709, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.079767349294387, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1068, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1066, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9552772808586762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9544235924932976, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9548502458649978, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.045576407506702415, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9526362823949955, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9530621367903442, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9552772808586762, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9971471853077316, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07438391718016768, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 578, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 576, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9203821656050956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.918918918918919, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9196499602227526, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.08108108108108109, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9171974522292994, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9157392686804452, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9164677804295943, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9203821656050956, "step_test_time": 9.390129623992834, "step_plot_time": 7.329462068999419, "step_validation_cycle_time": 21.295793627999956} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_290000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_290000.json new file mode 100644 index 0000000000000000000000000000000000000000..f8c1474aef74a40c3ee7a1b6914cbf2b2174af50 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_290000.json @@ -0,0 +1 @@ +{"epoch": 1.7170024689311363, "train": {"Classification_binary": 2.372060775756836, "Classification": 0.015297436155378819, "Regression_pt": 0.147856205701828, "Regression_eta": 0.021474631503224373, "Regression_sin_phi": 0.005816732533276081, "Regression_cos_phi": 0.010961873456835747, "Regression_energy": 0.17092230916023254, "Total": 6.2027482986450195, "Total_unweighted": 2.7443899642676115}, "train_group": {}, "valid": {"Classification_binary": 2.2621688842773438, "Classification": 0.015766505231248572, "Regression_pt": 0.12383153590750187, "Regression_eta": 0.012180873688231123, "Regression_sin_phi": 0.007016682878453681, "Regression_cos_phi": 0.0077491840149493925, "Regression_energy": 0.12882101789433906, "Total": 5.35824260305851, "Total_unweighted": 2.5575346838920674, "metrics/particle/class_1/efficiency": 0.9433423280715942, "metrics/particle/class_1/pid_accuracy": 0.9864148497581482, "metrics/particle/class_2/efficiency": 0.44978782534599304, "metrics/particle/class_2/pid_accuracy": 0.7882599830627441, "metrics/particle/class_3/efficiency": 0.8002442717552185, "metrics/particle/class_3/pid_accuracy": 0.9868294596672058, "metrics/particle/class_4/efficiency": 0.8350100517272949, "metrics/particle/class_4/pid_accuracy": 0.4891566336154938, "metrics/particle/class_5/efficiency": 0.9674796462059021, "metrics/particle/class_5/pid_accuracy": 0.6302521228790283, "metrics/particle/count/bias_mean": -0.0533333346247673, "metrics/particle/count/mae": 2.946666717529297, "metrics/particle/count/prediction_mean": 69.67466735839844, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.044439997524023056, "metrics/particle/event/energy_response_mean": 0.9933772087097168, "metrics/particle/event/met_abs_error": 5.020986557006836, "metrics/particle/event/scalar_pt_relative_abs_error": 0.036381106823682785, "metrics/particle/event/scalar_pt_response_mean": 1.002072811126709, "metrics/particle/event/vector_pt_closure": 0.04231512174010277, "metrics/particle/matched/delta_eta_abs_mean": 0.012050898745656013, "metrics/particle/matched/delta_phi_abs_mean": 0.01342862006276846, "metrics/particle/matched/delta_r_mean": 0.020127834752202034, "metrics/particle/matched/energy_relative_abs_mean": 0.0852871909737587, "metrics/particle/matched/pid_accuracy": 0.9713818430900574, "metrics/particle/matched/pt_relative_abs_mean": 0.08102011680603027, "metrics/particle/matching/duplicate_fraction": 0.027327004820108414, "metrics/particle/matching/efficiency": 0.8418999314308167, "metrics/particle/matching/f1": 0.8422220349311829, "metrics/particle/matching/purity": 0.8425443768501282, "metrics/particle/matching/target_coverage_dr0p05": 0.7848592400550842, "metrics/particle/matching/target_coverage_dr0p10": 0.8718066215515137, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.748680591583252, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8800482153892517}, "step_train_time": 5.670119524002075, "step_valid_time": 3.977820943982806, "step_total_time": 23345.629690408707, "elapsed_training_loop_seconds": 23345.629690408707, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0009247979684186, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06570845203902942, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9636608344549125, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9649595687331537, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9643097643097643, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03504043126684636, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9615902964959568, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.960942760942761, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9636608344549125, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0010147452133449, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0817188833238569, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1060, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1057, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9481216457960644, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9481216457960644, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9481216457960644, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0518783542039356, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.945438282647585, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.945438282647585, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.945438282647585, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9481216457960644, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0010294616312616, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07505857910217906, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 622, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 564, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 564, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8980891719745223, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9067524115755627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9024, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0932475884244373, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8980891719745223, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9067524115755627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9024, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8980891719745223, "step_test_time": 9.416577223979402, "step_plot_time": 7.38673124302295, "step_validation_cycle_time": 21.35705721899285} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_30000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_30000.json new file mode 100644 index 0000000000000000000000000000000000000000..cc589f8bd2794e6b1f5b7b63f0b32dcfd7c166d4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_30000.json @@ -0,0 +1 @@ +{"epoch": 0.1776209450618417, "train": {"Classification_binary": 4.060131072998047, "Classification": 0.01900441385805607, "Regression_pt": 0.21301941573619843, "Regression_eta": 0.010757984593510628, "Regression_sin_phi": 0.011623412370681763, "Regression_cos_phi": 0.014326894655823708, "Regression_energy": 0.20945176482200623, "Total": 9.131258964538574, "Total_unweighted": 4.538314959034324}, "train_group": {}, "valid": {"Classification_binary": 3.845541446766955, "Classification": 0.029242726082497453, "Regression_pt": 0.23145417964204829, "Regression_eta": 0.021576889017794993, "Regression_sin_phi": 0.011313130246832016, "Regression_cos_phi": 0.013443232850825533, "Regression_energy": 0.23874560822831822, "Total": 9.58433954766456, "Total_unweighted": 4.391317212835272, "metrics/particle/class_1/efficiency": 0.8633550405502319, "metrics/particle/class_1/pid_accuracy": 0.9754595756530762, "metrics/particle/class_2/efficiency": 0.20485620200634003, "metrics/particle/class_2/pid_accuracy": 0.3912543058395386, "metrics/particle/class_3/efficiency": 0.6881983280181885, "metrics/particle/class_3/pid_accuracy": 0.9721309542655945, "metrics/particle/class_4/efficiency": 0.800804853439331, "metrics/particle/class_4/pid_accuracy": 0.07035175710916519, "metrics/particle/class_5/efficiency": 0.9430894255638123, "metrics/particle/class_5/pid_accuracy": 0.2887931168079376, "metrics/particle/count/bias_mean": -0.4466666579246521, "metrics/particle/count/mae": 4.572000026702881, "metrics/particle/count/prediction_mean": 69.28133392333984, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.08787237852811813, "metrics/particle/event/energy_response_mean": 0.9743617177009583, "metrics/particle/event/met_abs_error": 10.073958396911621, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07636851817369461, "metrics/particle/event/scalar_pt_response_mean": 0.9874200820922852, "metrics/particle/event/vector_pt_closure": 0.08148522675037384, "metrics/particle/matched/delta_eta_abs_mean": 0.023317381739616394, "metrics/particle/matched/delta_phi_abs_mean": 0.01865311712026596, "metrics/particle/matched/delta_r_mean": 0.03315040096640587, "metrics/particle/matched/energy_relative_abs_mean": 0.09843263775110245, "metrics/particle/matched/pid_accuracy": 0.9474436640739441, "metrics/particle/matched/pt_relative_abs_mean": 0.10174717009067535, "metrics/particle/matching/duplicate_fraction": 0.03939493000507355, "metrics/particle/matching/efficiency": 0.7356776595115662, "metrics/particle/matching/f1": 0.7380415797233582, "metrics/particle/matching/purity": 0.7404206991195679, "metrics/particle/matching/target_coverage_dr0p05": 0.5921868085861206, "metrics/particle/matching/target_coverage_dr0p10": 0.7695425748825073, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6487685441970825, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8038855791091919}, "step_train_time": 5.119325160980225, "step_valid_time": 3.9121651320019737, "step_total_time": 2418.5461061000824, "elapsed_training_loop_seconds": 2418.5461061000824, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9792644869644893, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.1736770484618031, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1500, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1355, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1315, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9118438761776582, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9033333333333333, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9075686537173476, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.09666666666666666, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8849259757738897, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8766666666666667, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8807769591426657, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9118438761776582, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9924023091522973, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17469546751876708, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 984, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 965, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8801431127012522, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8809310653536258, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8805369127516779, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11906893464637422, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8631484794275492, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8639212175470009, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8635346756152126, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8801431127012522, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9951873090440468, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17447250201768028, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 621, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 502, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 494, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.7993630573248408, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8083735909822867, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8038430744595677, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.19162640901771336, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.7866242038216561, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.7954911433172303, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.7910328262610088, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.7993630573248408, "step_test_time": 8.862720289995195, "step_plot_time": 7.133814247004921, "step_validation_cycle_time": 20.359472799987998} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_300000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_300000.json new file mode 100644 index 0000000000000000000000000000000000000000..b6d9c341015b3b8c948511c17270d017c905f3ea --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_300000.json @@ -0,0 +1 @@ +{"epoch": 1.776209450618417, "train": {"Classification_binary": 2.113335132598877, "Classification": 0.016854839399456978, "Regression_pt": 0.10654591768980026, "Regression_eta": 0.013532711192965508, "Regression_sin_phi": 0.007266521919518709, "Regression_cos_phi": 0.007466491311788559, "Regression_energy": 0.10812778770923615, "Total": 4.820987224578857, "Total_unweighted": 2.373129401821643}, "train_group": {}, "valid": {"Classification_binary": 2.2601785862699466, "Classification": 0.01572423792899923, "Regression_pt": 0.12756070684879384, "Regression_eta": 0.012438412676466273, "Regression_sin_phi": 0.006815050510649985, "Regression_cos_phi": 0.007532296662634992, "Regression_energy": 0.13267040252685547, "Total": 5.4363228006565825, "Total_unweighted": 2.562919693424346, "metrics/particle/class_1/efficiency": 0.9455642104148865, "metrics/particle/class_1/pid_accuracy": 0.985313892364502, "metrics/particle/class_2/efficiency": 0.4561527669429779, "metrics/particle/class_2/pid_accuracy": 0.7881137132644653, "metrics/particle/class_3/efficiency": 0.7908354997634888, "metrics/particle/class_3/pid_accuracy": 0.9852427840232849, "metrics/particle/class_4/efficiency": 0.8672032356262207, "metrics/particle/class_4/pid_accuracy": 0.4802784323692322, "metrics/particle/class_5/efficiency": 0.9674796462059021, "metrics/particle/class_5/pid_accuracy": 0.6974790096282959, "metrics/particle/count/bias_mean": 0.08266666531562805, "metrics/particle/count/mae": 2.810666561126709, "metrics/particle/count/prediction_mean": 69.8106689453125, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04850241541862488, "metrics/particle/event/energy_response_mean": 1.0068186521530151, "metrics/particle/event/met_abs_error": 4.341088771820068, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037604302167892456, "metrics/particle/event/scalar_pt_response_mean": 1.0074833631515503, "metrics/particle/event/vector_pt_closure": 0.04141008108854294, "metrics/particle/matched/delta_eta_abs_mean": 0.013160008937120438, "metrics/particle/matched/delta_phi_abs_mean": 0.012369560077786446, "metrics/particle/matched/delta_r_mean": 0.020176803693175316, "metrics/particle/matched/energy_relative_abs_mean": 0.08580193668603897, "metrics/particle/matched/pid_accuracy": 0.9700812697410583, "metrics/particle/matched/pt_relative_abs_mean": 0.08074556291103363, "metrics/particle/matching/duplicate_fraction": 0.02792314440011978, "metrics/particle/matching/efficiency": 0.8398156762123108, "metrics/particle/matching/f1": 0.8393181562423706, "metrics/particle/matching/purity": 0.8388211727142334, "metrics/particle/matching/target_coverage_dr0p05": 0.7796007394790649, "metrics/particle/matching/target_coverage_dr0p10": 0.8703151345252991, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7463668584823608, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8801246881484985}, "step_train_time": 5.109952688217163, "step_valid_time": 3.9559196879854426, "step_total_time": 24137.76755452156, "elapsed_training_loop_seconds": 24137.76755452156, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0013570262888343, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0691277682156145, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1426, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9596231493943472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9635135135135136, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.961564396493594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03648648648648649, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9576043068640646, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9614864864864865, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.959541469993257, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9596231493943472, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0029310724038774, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07148329728565539, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1115, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1065, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1063, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9525939177101967, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9551569506726457, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9538737124944021, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04484304932735426, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9508050089445438, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9533632286995516, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9520824003582624, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9525939177101967, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0085720114676318, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08420136812057488, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 583, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 581, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.928343949044586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9298245614035088, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9290836653386454, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07017543859649122, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9251592356687898, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9266347687400319, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9258964143426295, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.928343949044586, "step_test_time": 9.393195898999693, "step_plot_time": 7.389807361993007, "step_validation_cycle_time": 21.265474498010008} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_310000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_310000.json new file mode 100644 index 0000000000000000000000000000000000000000..7bd54a192de5a0b58295530b0cb4f1ea89a631f3 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_310000.json @@ -0,0 +1 @@ +{"epoch": 1.8354164323056974, "train": {"Classification_binary": 2.3010854721069336, "Classification": 0.01802525296807289, "Regression_pt": 0.1827276349067688, "Regression_eta": 0.011290770955383778, "Regression_sin_phi": 0.006726011633872986, "Regression_cos_phi": 0.0039219302125275135, "Regression_energy": 0.18308351933956146, "Total": 6.637236595153809, "Total_unweighted": 2.706860592123121}, "train_group": {}, "valid": {"Classification_binary": 2.2397389513380985, "Classification": 0.015483773769216334, "Regression_pt": 0.12426193724287317, "Regression_eta": 0.01207529991231066, "Regression_sin_phi": 0.006813898365548316, "Regression_cos_phi": 0.007632227654152728, "Regression_energy": 0.12949020304578415, "Total": 5.341086529670878, "Total_unweighted": 2.5354962913279846, "metrics/particle/class_1/efficiency": 0.94707190990448, "metrics/particle/class_1/pid_accuracy": 0.9852953553199768, "metrics/particle/class_2/efficiency": 0.4625176787376404, "metrics/particle/class_2/pid_accuracy": 0.7951070070266724, "metrics/particle/class_3/efficiency": 0.8053105473518372, "metrics/particle/class_3/pid_accuracy": 0.9855080842971802, "metrics/particle/class_4/efficiency": 0.8350100517272949, "metrics/particle/class_4/pid_accuracy": 0.47228914499282837, "metrics/particle/class_5/efficiency": 0.9796748161315918, "metrics/particle/class_5/pid_accuracy": 0.6846473217010498, "metrics/particle/count/bias_mean": 1.3346666097640991, "metrics/particle/count/mae": 2.9826667308807373, "metrics/particle/count/prediction_mean": 71.06266784667969, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04692710191011429, "metrics/particle/event/energy_response_mean": 1.0125751495361328, "metrics/particle/event/met_abs_error": 4.423859119415283, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037956930696964264, "metrics/particle/event/scalar_pt_response_mean": 1.0169600248336792, "metrics/particle/event/vector_pt_closure": 0.03994886577129364, "metrics/particle/matched/delta_eta_abs_mean": 0.012446960434317589, "metrics/particle/matched/delta_phi_abs_mean": 0.012889486737549305, "metrics/particle/matched/delta_r_mean": 0.019968463107943535, "metrics/particle/matched/energy_relative_abs_mean": 0.0858081579208374, "metrics/particle/matched/pid_accuracy": 0.9705132246017456, "metrics/particle/matched/pt_relative_abs_mean": 0.08245665580034256, "metrics/particle/matching/duplicate_fraction": 0.030545810237526894, "metrics/particle/matching/efficiency": 0.8469290137290955, "metrics/particle/matching/f1": 0.8389003276824951, "metrics/particle/matching/purity": 0.8310223817825317, "metrics/particle/matching/target_coverage_dr0p05": 0.7894867658615112, "metrics/particle/matching/target_coverage_dr0p10": 0.8784227967262268, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.749674916267395, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8864540457725525}, "step_train_time": 5.730425596237183, "step_valid_time": 3.9775895139900967, "step_total_time": 24956.094119787216, "elapsed_training_loop_seconds": 24956.094119787216, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0060402241247524, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06355387032934567, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1424, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9622895622895623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9619656681252103, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03771043771043771, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9582772543741588, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.958922558922559, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9585997980477954, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9616419919246298, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.005990346019558, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07160616155325517, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1072, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1071, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9588550983899821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9545859305431879, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9567157518964748, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04541406945681211, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9579606440071556, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9536954585930543, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9558232931726908, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9588550983899821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0101014946742042, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.08319521571326072, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 587, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 587, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9347133757961783, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9362041467304625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9354581673306773, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06379585326953748, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9347133757961783, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9362041467304625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9354581673306773, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9347133757961783, "step_test_time": 9.48231513099745, "step_plot_time": 7.32201695398544, "step_validation_cycle_time": 21.342184045992326} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_320000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_320000.json new file mode 100644 index 0000000000000000000000000000000000000000..ff9ef33b702ed1025fd842735981865c57db7e0f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_320000.json @@ -0,0 +1 @@ +{"epoch": 1.894623413992978, "train": {"Classification_binary": 2.497087240219116, "Classification": 0.014372955076396465, "Regression_pt": 0.1647549569606781, "Regression_eta": 0.046750508248806, "Regression_sin_phi": 0.008638440631330013, "Regression_cos_phi": 0.008573352359235287, "Regression_energy": 0.16311776638031006, "Total": 6.579529762268066, "Total_unweighted": 2.903295219875872}, "train_group": {}, "valid": {"Classification_binary": 2.2163939780377326, "Classification": 0.015326596320943629, "Regression_pt": 0.12410345483333507, "Regression_eta": 0.012186856980019428, "Regression_sin_phi": 0.006710302956560825, "Regression_cos_phi": 0.007445476790691944, "Regression_energy": 0.12896400816897127, "Total": 5.307517193733378, "Total_unweighted": 2.511130674088255, "metrics/particle/class_1/efficiency": 0.9460006356239319, "metrics/particle/class_1/pid_accuracy": 0.9848592877388, "metrics/particle/class_2/efficiency": 0.44436585903167725, "metrics/particle/class_2/pid_accuracy": 0.8010610342025757, "metrics/particle/class_3/efficiency": 0.7963541150093079, "metrics/particle/class_3/pid_accuracy": 0.9850610494613647, "metrics/particle/class_4/efficiency": 0.8370221257209778, "metrics/particle/class_4/pid_accuracy": 0.4927884638309479, "metrics/particle/class_5/efficiency": 0.9715447425842285, "metrics/particle/class_5/pid_accuracy": 0.6987447738647461, "metrics/particle/count/bias_mean": -0.2866666615009308, "metrics/particle/count/mae": 2.8413333892822266, "metrics/particle/count/prediction_mean": 69.44132995605469, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04451953247189522, "metrics/particle/event/energy_response_mean": 1.0029946565628052, "metrics/particle/event/met_abs_error": 4.541224479675293, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03528197109699249, "metrics/particle/event/scalar_pt_response_mean": 1.004422903060913, "metrics/particle/event/vector_pt_closure": 0.040756043046712875, "metrics/particle/matched/delta_eta_abs_mean": 0.012950372882187366, "metrics/particle/matched/delta_phi_abs_mean": 0.013581492938101292, "metrics/particle/matched/delta_r_mean": 0.02091696485877037, "metrics/particle/matched/energy_relative_abs_mean": 0.0856039747595787, "metrics/particle/matched/pid_accuracy": 0.970855712890625, "metrics/particle/matched/pt_relative_abs_mean": 0.08101396262645721, "metrics/particle/matching/duplicate_fraction": 0.027207618579268456, "metrics/particle/matching/efficiency": 0.8411350846290588, "metrics/particle/matching/f1": 0.8428676724433899, "metrics/particle/matching/purity": 0.844607412815094, "metrics/particle/matching/target_coverage_dr0p05": 0.7745525240898132, "metrics/particle/matching/target_coverage_dr0p10": 0.8701812624931335, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7475715279579163, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.879359781742096}, "step_train_time": 4.958007335662842, "step_valid_time": 3.9738135740044527, "step_total_time": 25765.395270824432, "elapsed_training_loop_seconds": 25765.395270824432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0040668403224755, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06409914561010976, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1425, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1419, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9589502018842531, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9583053127101546, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.958627648839556, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04169468728984533, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9549125168236877, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9542703429724277, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9545913218970736, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9589502018842531, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0016474328044853, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07346658317453947, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1069, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1066, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04382826475849732, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561717352415027, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0048479675247677, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0742680453790392, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 580, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 579, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9235668789808917, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.928, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9257781324820431, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.072, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9264, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9241819632881085, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9235668789808917, "step_test_time": 9.492855040996801, "step_plot_time": 7.4116925449925475, "step_validation_cycle_time": 21.449888033006573} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_330000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_330000.json new file mode 100644 index 0000000000000000000000000000000000000000..39862e386adc88bb201fe17b1aee0380dbe797a0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_330000.json @@ -0,0 +1 @@ +{"epoch": 1.9538303956802585, "train": {"Classification_binary": 2.358675241470337, "Classification": 0.02132413536310196, "Regression_pt": 0.12601736187934875, "Regression_eta": 0.01363727729767561, "Regression_sin_phi": 0.013753808103501797, "Regression_cos_phi": 0.013827512972056866, "Regression_energy": 0.1240871474146843, "Total": 5.604887008666992, "Total_unweighted": 2.671322484500706}, "train_group": {}, "valid": {"Classification_binary": 2.2171530216298203, "Classification": 0.015311805491751812, "Regression_pt": 0.12131434298576192, "Regression_eta": 0.011801578897110959, "Regression_sin_phi": 0.0068244325353744185, "Regression_cos_phi": 0.007443195327799371, "Regression_energy": 0.12631994612673494, "Total": 5.247874158494016, "Total_unweighted": 2.5061683229943537, "metrics/particle/class_1/efficiency": 0.9429455399513245, "metrics/particle/class_1/pid_accuracy": 0.9852309823036194, "metrics/particle/class_2/efficiency": 0.45803865790367126, "metrics/particle/class_2/pid_accuracy": 0.800823450088501, "metrics/particle/class_3/efficiency": 0.800199031829834, "metrics/particle/class_3/pid_accuracy": 0.9880723357200623, "metrics/particle/class_4/efficiency": 0.8390341997146606, "metrics/particle/class_4/pid_accuracy": 0.48441246151924133, "metrics/particle/class_5/efficiency": 0.9796748161315918, "metrics/particle/class_5/pid_accuracy": 0.7053942084312439, "metrics/particle/count/bias_mean": -0.20000000298023224, "metrics/particle/count/mae": 2.797333240509033, "metrics/particle/count/prediction_mean": 69.52799987792969, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04365310072898865, "metrics/particle/event/energy_response_mean": 1.002323865890503, "metrics/particle/event/met_abs_error": 4.268577575683594, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03554892912507057, "metrics/particle/event/scalar_pt_response_mean": 1.0071287155151367, "metrics/particle/event/vector_pt_closure": 0.03854918107390404, "metrics/particle/matched/delta_eta_abs_mean": 0.012113508768379688, "metrics/particle/matched/delta_phi_abs_mean": 0.012160792015492916, "metrics/particle/matched/delta_r_mean": 0.01917056366801262, "metrics/particle/matched/energy_relative_abs_mean": 0.08455190807580948, "metrics/particle/matched/pid_accuracy": 0.9719681143760681, "metrics/particle/matched/pt_relative_abs_mean": 0.08061692118644714, "metrics/particle/matching/duplicate_fraction": 0.026828519999980927, "metrics/particle/matching/efficiency": 0.8424544930458069, "metrics/particle/matching/f1": 0.8436644077301025, "metrics/particle/matching/purity": 0.8448778390884399, "metrics/particle/matching/target_coverage_dr0p05": 0.7952806949615479, "metrics/particle/matching/target_coverage_dr0p10": 0.8724376559257507, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7494263648986816, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8802202939987183}, "step_train_time": 5.33545446395874, "step_valid_time": 3.9726590310165193, "step_total_time": 26576.312781095505, "elapsed_training_loop_seconds": 26576.312781095505, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038982744878766, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06298096692264976, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1423, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1416, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9576043068640646, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.956960322797579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9572822065253952, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04303967720242098, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9528936742934051, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9522528581035642, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9525731584258325, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9576043068640646, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036260756305553, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07168338933191665, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1069, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1066, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04382826475849732, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9534883720930233, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561717352415027, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0067538727266758, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07252501519962729, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 579, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 579, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9249201277955271, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9234449760765551, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07507987220447285, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9249201277955271, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9234449760765551, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9219745222929936, "step_test_time": 9.446888480015332, "step_plot_time": 7.350398718001088, "step_validation_cycle_time": 21.342620829993393} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_340000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_340000.json new file mode 100644 index 0000000000000000000000000000000000000000..255153e8b7cc44246edf8bf41b5ebd30bda7b265 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_340000.json @@ -0,0 +1 @@ +{"epoch": 2.013037377367539, "train": {"Classification_binary": 2.15244722366333, "Classification": 0.013773932121694088, "Regression_pt": 0.12231969833374023, "Regression_eta": 0.004519453272223473, "Regression_sin_phi": 0.005741870030760765, "Regression_cos_phi": 0.00429563270881772, "Regression_energy": 0.11932120472192764, "Total": 5.0526442527771, "Total_unweighted": 2.422419014852494}, "train_group": {}, "valid": {"Classification_binary": 2.192334601219664, "Classification": 0.015209181511655767, "Regression_pt": 0.12085328203566531, "Regression_eta": 0.011862037029672177, "Regression_sin_phi": 0.006759255490404494, "Regression_cos_phi": 0.007400479088438318, "Regression_energy": 0.12555381085010284, "Total": 5.208409410841922, "Total_unweighted": 2.479972647225603, "metrics/particle/class_1/efficiency": 0.9452468156814575, "metrics/particle/class_1/pid_accuracy": 0.9857286810874939, "metrics/particle/class_2/efficiency": 0.4655822813510895, "metrics/particle/class_2/pid_accuracy": 0.8025316596031189, "metrics/particle/class_3/efficiency": 0.8030940294265747, "metrics/particle/class_3/pid_accuracy": 0.9869325160980225, "metrics/particle/class_4/efficiency": 0.849094569683075, "metrics/particle/class_4/pid_accuracy": 0.49526065587997437, "metrics/particle/class_5/efficiency": 0.9756097793579102, "metrics/particle/class_5/pid_accuracy": 0.699999988079071, "metrics/particle/count/bias_mean": 0.2933333218097687, "metrics/particle/count/mae": 2.74399995803833, "metrics/particle/count/prediction_mean": 70.02133178710938, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04236694425344467, "metrics/particle/event/energy_response_mean": 0.9961525201797485, "metrics/particle/event/met_abs_error": 4.078068256378174, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03345492482185364, "metrics/particle/event/scalar_pt_response_mean": 0.9977502226829529, "metrics/particle/event/vector_pt_closure": 0.03841417282819748, "metrics/particle/matched/delta_eta_abs_mean": 0.012822899967432022, "metrics/particle/matched/delta_phi_abs_mean": 0.012846049852669239, "metrics/particle/matched/delta_r_mean": 0.02021452970802784, "metrics/particle/matched/energy_relative_abs_mean": 0.08525525778532028, "metrics/particle/matched/pid_accuracy": 0.9717969298362732, "metrics/particle/matched/pt_relative_abs_mean": 0.08064907044172287, "metrics/particle/matching/duplicate_fraction": 0.025477949529886246, "metrics/particle/matching/efficiency": 0.8454757332801819, "metrics/particle/matching/f1": 0.8437011241912842, "metrics/particle/matching/purity": 0.8419339060783386, "metrics/particle/matching/target_coverage_dr0p05": 0.7878231406211853, "metrics/particle/matching/target_coverage_dr0p10": 0.8751912117004395, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.75334632396698, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8844462037086487}, "step_train_time": 5.137289762496948, "step_valid_time": 3.966631415009033, "step_total_time": 27393.68043899536, "elapsed_training_loop_seconds": 27393.68043899536, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0023507974675385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06252234052197603, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1421, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9636608344549125, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9636608344549125, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9636608344549125, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03633916554508748, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9562584118438762, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9562584118438762, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9562584118438762, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9636608344549125, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9974945363560721, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07350132890237093, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1076, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1070, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.962432915921288, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.963294538943599, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9628635346756153, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03670546105640107, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9570661896243292, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9579230080572964, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9574944071588367, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.962432915921288, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0007202708253027, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06751394106356234, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 579, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 578, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9249201277955271, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9234449760765551, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07507987220447285, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9203821656050956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9233226837060703, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9218500797448166, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9219745222929936, "step_test_time": 9.47014459001366, "step_plot_time": 7.366924142988864, "step_validation_cycle_time": 21.378553749993443} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_350000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_350000.json new file mode 100644 index 0000000000000000000000000000000000000000..944f10395955bdf0efdac780d38135d71dac7031 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_350000.json @@ -0,0 +1 @@ +{"epoch": 2.0722443590548196, "train": {"Classification_binary": 2.2270936965942383, "Classification": 0.012207728810608387, "Regression_pt": 0.09124631434679031, "Regression_eta": 0.0039598373696208, "Regression_sin_phi": 0.0035828177351504564, "Regression_cos_phi": 0.006004547234624624, "Regression_energy": 0.09241307526826859, "Total": 4.458407878875732, "Total_unweighted": 2.4365080173593014}, "train_group": {}, "valid": {"Classification_binary": 2.1833871070374835, "Classification": 0.015132115242329049, "Regression_pt": 0.11969451701387446, "Regression_eta": 0.01203830952339984, "Regression_sin_phi": 0.00658545215079125, "Regression_cos_phi": 0.00718975193957065, "Regression_energy": 0.12470307248703977, "Total": 5.174208620761303, "Total_unweighted": 2.468730325394488, "metrics/particle/class_1/efficiency": 0.9469131827354431, "metrics/particle/class_1/pid_accuracy": 0.9857956767082214, "metrics/particle/class_2/efficiency": 0.4693540930747986, "metrics/particle/class_2/pid_accuracy": 0.8091411590576172, "metrics/particle/class_3/efficiency": 0.8063057065010071, "metrics/particle/class_3/pid_accuracy": 0.9878261089324951, "metrics/particle/class_4/efficiency": 0.8289738297462463, "metrics/particle/class_4/pid_accuracy": 0.5097087621688843, "metrics/particle/class_5/efficiency": 0.9756097793579102, "metrics/particle/class_5/pid_accuracy": 0.6916666626930237, "metrics/particle/count/bias_mean": 0.47600001096725464, "metrics/particle/count/mae": 2.828000068664551, "metrics/particle/count/prediction_mean": 70.2040023803711, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04837736859917641, "metrics/particle/event/energy_response_mean": 1.0113029479980469, "metrics/particle/event/met_abs_error": 4.485816478729248, "metrics/particle/event/scalar_pt_relative_abs_error": 0.037171296775341034, "metrics/particle/event/scalar_pt_response_mean": 1.0129393339157104, "metrics/particle/event/vector_pt_closure": 0.0399642214179039, "metrics/particle/matched/delta_eta_abs_mean": 0.012568913400173187, "metrics/particle/matched/delta_phi_abs_mean": 0.012217328883707523, "metrics/particle/matched/delta_r_mean": 0.019621502608060837, "metrics/particle/matched/energy_relative_abs_mean": 0.08749672770500183, "metrics/particle/matched/pid_accuracy": 0.972662091255188, "metrics/particle/matched/pt_relative_abs_mean": 0.0819837674498558, "metrics/particle/matching/duplicate_fraction": 0.02835545875132084, "metrics/particle/matching/efficiency": 0.847751259803772, "metrics/particle/matching/f1": 0.8448675274848938, "metrics/particle/matching/purity": 0.8420032858848572, "metrics/particle/matching/target_coverage_dr0p05": 0.7976900935173035, "metrics/particle/matching/target_coverage_dr0p10": 0.878021240234375, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7501147389411926, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8849242925643921}, "step_train_time": 5.260523080825806, "step_valid_time": 3.962218312022742, "step_total_time": 28205.46095585823, "elapsed_training_loop_seconds": 28205.46095585823, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0078296040386623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06397458904986253, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1488, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1437, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9670255720053835, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9657258064516129, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9663752521856086, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.034274193548387094, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9609690444145357, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9596774193548387, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9603227975790182, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9670255720053835, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.008650599723326, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07159443552907052, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1072, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1069, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9588550983899821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9545859305431879, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9567157518964748, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04541406945681211, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9519145146927872, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9540383757251227, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9588550983899821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0096170370548412, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07079060634556578, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 623, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 579, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 577, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9219745222929936, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9293739967897271, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9256594724220624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07062600321027288, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9187898089171974, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9261637239165329, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9224620303756994, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9219745222929936, "step_test_time": 9.458778044034261, "step_plot_time": 7.3770622329902835, "step_validation_cycle_time": 21.37675183697138} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_360000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_360000.json new file mode 100644 index 0000000000000000000000000000000000000000..612769a9d79a45e4f5227105da2292b829adff13 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_360000.json @@ -0,0 +1 @@ +{"epoch": 2.1314513407421005, "train": {"Classification_binary": 2.135683298110962, "Classification": 0.014698510058224201, "Regression_pt": 0.1439981907606125, "Regression_eta": 0.012800382450222969, "Regression_sin_phi": 0.006151237059384584, "Regression_cos_phi": 0.00574658764526248, "Regression_energy": 0.1409202516078949, "Total": 5.5676984786987305, "Total_unweighted": 2.4599984576925635}, "train_group": {}, "valid": {"Classification_binary": 2.1711911140604223, "Classification": 0.015074244204987871, "Regression_pt": 0.12100958316884142, "Regression_eta": 0.012030267969090888, "Regression_sin_phi": 0.006579089037915493, "Regression_cos_phi": 0.007260244577489001, "Regression_energy": 0.12603508157933013, "Total": 5.190442998358544, "Total_unweighted": 2.4591796245980775, "metrics/particle/class_1/efficiency": 0.9481828212738037, "metrics/particle/class_1/pid_accuracy": 0.9864423871040344, "metrics/particle/class_2/efficiency": 0.4707685112953186, "metrics/particle/class_2/pid_accuracy": 0.8012018203735352, "metrics/particle/class_3/efficiency": 0.8062604665756226, "metrics/particle/class_3/pid_accuracy": 0.9895646572113037, "metrics/particle/class_4/efficiency": 0.8329979777336121, "metrics/particle/class_4/pid_accuracy": 0.5169082283973694, "metrics/particle/class_5/efficiency": 0.9878048896789551, "metrics/particle/class_5/pid_accuracy": 0.7119341492652893, "metrics/particle/count/bias_mean": 0.36533331871032715, "metrics/particle/count/mae": 2.8239998817443848, "metrics/particle/count/prediction_mean": 70.09333038330078, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04118866100907326, "metrics/particle/event/energy_response_mean": 0.9863907098770142, "metrics/particle/event/met_abs_error": 4.0157575607299805, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0307949036359787, "metrics/particle/event/scalar_pt_response_mean": 0.9968683123588562, "metrics/particle/event/vector_pt_closure": 0.03694498538970947, "metrics/particle/matched/delta_eta_abs_mean": 0.01224393118172884, "metrics/particle/matched/delta_phi_abs_mean": 0.012111064977943897, "metrics/particle/matched/delta_r_mean": 0.019217826426029205, "metrics/particle/matched/energy_relative_abs_mean": 0.08531926572322845, "metrics/particle/matched/pid_accuracy": 0.9734766483306885, "metrics/particle/matched/pt_relative_abs_mean": 0.08076411485671997, "metrics/particle/matching/duplicate_fraction": 0.025984400883316994, "metrics/particle/matching/efficiency": 0.8485543727874756, "metrics/particle/matching/f1": 0.8463372588157654, "metrics/particle/matching/purity": 0.8441316485404968, "metrics/particle/matching/target_coverage_dr0p05": 0.796542763710022, "metrics/particle/matching/target_coverage_dr0p10": 0.8774667382240295, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7526196837425232, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8858038783073425}, "step_train_time": 5.395326852798462, "step_valid_time": 4.025014357001055, "step_total_time": 29025.851824760437, "elapsed_training_loop_seconds": 29025.851824760437, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0002290994971395, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06501809377335599, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1490, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1438, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9676985195154778, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9651006711409396, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9663978494623656, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0348993288590604, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9636608344549125, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9610738255033557, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9623655913978495, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9676985195154778, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.998452518002592, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07115194077343423, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1072, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1069, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9588550983899821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9579982126899017, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9584264640143049, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0420017873100983, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9553172475424486, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9557443004023245, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9588550983899821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9967772089524345, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06547175919404369, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 578, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 578, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9203821656050956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9233226837060703, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9218500797448166, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07667731629392971, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9203821656050956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9233226837060703, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9218500797448166, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9203821656050956, "step_test_time": 9.950540434976574, "step_plot_time": 7.437420705973636, "step_validation_cycle_time": 21.940145445987582} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_370000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_370000.json new file mode 100644 index 0000000000000000000000000000000000000000..ed36f3f179a55f41f509038a39f20043b78510a6 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_370000.json @@ -0,0 +1 @@ +{"epoch": 2.190658322429381, "train": {"Classification_binary": 2.040651798248291, "Classification": 0.016316020861268044, "Regression_pt": 0.17126886546611786, "Regression_eta": 0.015936093404889107, "Regression_sin_phi": 0.006106366403400898, "Regression_cos_phi": 0.007638942915946245, "Regression_energy": 0.17140941321849823, "Total": 6.141051769256592, "Total_unweighted": 2.4293275005184114}, "train_group": {}, "valid": {"Classification_binary": 2.146063784335522, "Classification": 0.014939908017503454, "Regression_pt": 0.11809647336919257, "Regression_eta": 0.011659223982628355, "Regression_sin_phi": 0.006614045893892329, "Regression_cos_phi": 0.007065269541233144, "Regression_energy": 0.12309214409361495, "Total": 5.097420875062334, "Total_unweighted": 2.427530849233587, "metrics/particle/class_1/efficiency": 0.950166642665863, "metrics/particle/class_1/pid_accuracy": 0.9870135188102722, "metrics/particle/class_2/efficiency": 0.46888259053230286, "metrics/particle/class_2/pid_accuracy": 0.8230266571044922, "metrics/particle/class_3/efficiency": 0.8187451958656311, "metrics/particle/class_3/pid_accuracy": 0.987790048122406, "metrics/particle/class_4/efficiency": 0.8611670136451721, "metrics/particle/class_4/pid_accuracy": 0.5373831987380981, "metrics/particle/class_5/efficiency": 0.9837398529052734, "metrics/particle/class_5/pid_accuracy": 0.6859503984451294, "metrics/particle/count/bias_mean": 0.8693333268165588, "metrics/particle/count/mae": 2.7733333110809326, "metrics/particle/count/prediction_mean": 70.59733581542969, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04603174701333046, "metrics/particle/event/energy_response_mean": 1.0093364715576172, "metrics/particle/event/met_abs_error": 4.093777656555176, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03403867036104202, "metrics/particle/event/scalar_pt_response_mean": 1.0071678161621094, "metrics/particle/event/vector_pt_closure": 0.03801403194665909, "metrics/particle/matched/delta_eta_abs_mean": 0.011588373221457005, "metrics/particle/matched/delta_phi_abs_mean": 0.01154045108705759, "metrics/particle/matched/delta_r_mean": 0.018287107348442078, "metrics/particle/matched/energy_relative_abs_mean": 0.08470410853624344, "metrics/particle/matched/pid_accuracy": 0.9740980267524719, "metrics/particle/matched/pt_relative_abs_mean": 0.07954845577478409, "metrics/particle/matching/duplicate_fraction": 0.026157740503549576, "metrics/particle/matching/efficiency": 0.8548837304115295, "metrics/particle/matching/f1": 0.849587619304657, "metrics/particle/matching/purity": 0.8443567156791687, "metrics/particle/matching/target_coverage_dr0p05": 0.8088763952255249, "metrics/particle/matching/target_coverage_dr0p10": 0.8825722932815552, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7612245678901672, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.891253650188446}, "step_train_time": 4.803480386734009, "step_valid_time": 3.9878438399755396, "step_total_time": 29838.22810983658, "elapsed_training_loop_seconds": 29838.22810983658, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000572138834857, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.059937187322337104, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1481, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1432, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9636608344549125, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9669142471303174, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9652847994607348, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03308575286968265, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9609690444145357, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9642133693450371, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9625884732052579, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9636608344549125, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036759528078671, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07187616345804937, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1115, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1076, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1074, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.962432915921288, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9650224215246637, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9637259292431706, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03497757847533632, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960644007155635, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9632286995515695, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9619346171070309, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.962432915921288, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.007697287344649, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.0669228201801022, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 584, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 584, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9299363057324841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9299363057324841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9299363057324841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07006369426751592, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9299363057324841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9299363057324841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9299363057324841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9299363057324841, "step_test_time": 9.518720104999375, "step_plot_time": 7.335781956964638, "step_validation_cycle_time": 21.409669129003305} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_380000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_380000.json new file mode 100644 index 0000000000000000000000000000000000000000..7b24e3d532fa571eb7d640200fee8dab64d9da13 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_380000.json @@ -0,0 +1 @@ +{"epoch": 2.2498653041166614, "train": {"Classification_binary": 2.200526714324951, "Classification": 0.011282755061984062, "Regression_pt": 0.10954749584197998, "Regression_eta": 0.003919896204024553, "Regression_sin_phi": 0.014856627210974693, "Regression_cos_phi": 0.0033726149704307318, "Regression_energy": 0.10554519295692444, "Total": 4.851729869842529, "Total_unweighted": 2.4490512965712696}, "train_group": {}, "valid": {"Classification_binary": 2.1428393100170378, "Classification": 0.01489749106955021, "Regression_pt": 0.11855190358263382, "Regression_eta": 0.011615743028356674, "Regression_sin_phi": 0.0066519500093257175, "Regression_cos_phi": 0.0072676913535341305, "Regression_energy": 0.12335849315562147, "Total": 5.103153634578623, "Total_unweighted": 2.4251825822160593, "metrics/particle/class_1/efficiency": 0.9503650069236755, "metrics/particle/class_1/pid_accuracy": 0.986223042011261, "metrics/particle/class_2/efficiency": 0.4698255658149719, "metrics/particle/class_2/pid_accuracy": 0.8138484954833984, "metrics/particle/class_3/efficiency": 0.8100601434707642, "metrics/particle/class_3/pid_accuracy": 0.9872124195098877, "metrics/particle/class_4/efficiency": 0.8651911616325378, "metrics/particle/class_4/pid_accuracy": 0.5325581431388855, "metrics/particle/class_5/efficiency": 0.9715447425842285, "metrics/particle/class_5/pid_accuracy": 0.7196652889251709, "metrics/particle/count/bias_mean": 0.18400000035762787, "metrics/particle/count/mae": 2.6106667518615723, "metrics/particle/count/prediction_mean": 69.91200256347656, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04425539821386337, "metrics/particle/event/energy_response_mean": 1.0041913986206055, "metrics/particle/event/met_abs_error": 4.454237461090088, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0348300077021122, "metrics/particle/event/scalar_pt_response_mean": 1.006803035736084, "metrics/particle/event/vector_pt_closure": 0.03949817270040512, "metrics/particle/matched/delta_eta_abs_mean": 0.01180616207420826, "metrics/particle/matched/delta_phi_abs_mean": 0.01153874397277832, "metrics/particle/matched/delta_r_mean": 0.0184776708483696, "metrics/particle/matched/energy_relative_abs_mean": 0.0844275951385498, "metrics/particle/matched/pid_accuracy": 0.9730925559997559, "metrics/particle/matched/pt_relative_abs_mean": 0.07987698912620544, "metrics/particle/matching/duplicate_fraction": 0.026185300201177597, "metrics/particle/matching/efficiency": 0.851365327835083, "metrics/particle/matching/f1": 0.8502435088157654, "metrics/particle/matching/purity": 0.8491246104240417, "metrics/particle/matching/target_coverage_dr0p05": 0.8051093816757202, "metrics/particle/matching/target_coverage_dr0p10": 0.8784227967262268, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7572472095489502, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8874675035476685}, "step_train_time": 5.290137052536011, "step_valid_time": 4.014963990019169, "step_total_time": 30656.292862653732, "elapsed_training_loop_seconds": 30656.292862653732, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002878611381104, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06246929583435201, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1424, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1418, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9582772543741588, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.958922558922559, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9585997980477954, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04107744107744108, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9542395693135935, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9548821548821549, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9545607539548974, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9582772543741588, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.003183436956328, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06916261237162424, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1072, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1070, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9588550983899821, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9579982126899017, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9584264640143049, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0420017873100983, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9570661896243292, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9562109025915997, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9566383549396513, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9588550983899821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0047612202054983, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06634197491389926, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 581, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 580, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9251592356687898, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9281150159744409, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9266347687400319, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.07188498402555911, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9235668789808917, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9265175718849841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9250398724082934, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9251592356687898, "step_test_time": 10.74145699897781, "step_plot_time": 7.427677702973597, "step_validation_cycle_time": 22.729797294014134} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_390000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_390000.json new file mode 100644 index 0000000000000000000000000000000000000000..fd0a9e4645363b753beacc0dc393e5b67c481fd2 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_390000.json @@ -0,0 +1 @@ +{"epoch": 2.309072285803942, "train": {"Classification_binary": 2.033508777618408, "Classification": 0.012427626177668571, "Regression_pt": 0.08549677580595016, "Regression_eta": 0.007562263403087854, "Regression_sin_phi": 0.005051902495324612, "Regression_cos_phi": 0.005564417224377394, "Regression_energy": 0.08237161487340927, "Total": 4.128512859344482, "Total_unweighted": 2.231983377598226}, "train_group": {}, "valid": {"Classification_binary": 2.1308144102705286, "Classification": 0.014734654984575636, "Regression_pt": 0.11534040532213577, "Regression_eta": 0.011710251899475747, "Regression_sin_phi": 0.006479839061168914, "Regression_cos_phi": 0.007115934757476157, "Regression_energy": 0.12047799090121655, "Total": 5.020285748420878, "Total_unweighted": 2.4066734871965774, "metrics/particle/class_1/efficiency": 0.948460578918457, "metrics/particle/class_1/pid_accuracy": 0.9865718483924866, "metrics/particle/class_2/efficiency": 0.47972655296325684, "metrics/particle/class_2/pid_accuracy": 0.8142505884170532, "metrics/particle/class_3/efficiency": 0.8095625638961792, "metrics/particle/class_3/pid_accuracy": 0.9879868030548096, "metrics/particle/class_4/efficiency": 0.849094569683075, "metrics/particle/class_4/pid_accuracy": 0.5260663628578186, "metrics/particle/class_5/efficiency": 0.9796748161315918, "metrics/particle/class_5/pid_accuracy": 0.7344398498535156, "metrics/particle/count/bias_mean": 0.3973333239555359, "metrics/particle/count/mae": 2.7279999256134033, "metrics/particle/count/prediction_mean": 70.12533569335938, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04387855902314186, "metrics/particle/event/energy_response_mean": 1.0100107192993164, "metrics/particle/event/met_abs_error": 3.9649577140808105, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0346592552959919, "metrics/particle/event/scalar_pt_response_mean": 1.0093144178390503, "metrics/particle/event/vector_pt_closure": 0.03785603493452072, "metrics/particle/matched/delta_eta_abs_mean": 0.011860709637403488, "metrics/particle/matched/delta_phi_abs_mean": 0.011560003273189068, "metrics/particle/matched/delta_r_mean": 0.018503259867429733, "metrics/particle/matched/energy_relative_abs_mean": 0.08522636443376541, "metrics/particle/matched/pid_accuracy": 0.9735280871391296, "metrics/particle/matched/pt_relative_abs_mean": 0.07985740900039673, "metrics/particle/matching/duplicate_fraction": 0.02671407349407673, "metrics/particle/matching/efficiency": 0.8509255051612854, "metrics/particle/matching/f1": 0.8485079407691956, "metrics/particle/matching/purity": 0.8461041450500488, "metrics/particle/matching/target_coverage_dr0p05": 0.8048607707023621, "metrics/particle/matching/target_coverage_dr0p10": 0.8797039985656738, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7552394270896912, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8867599964141846}, "step_train_time": 5.70368218421936, "step_valid_time": 4.010057809995487, "step_total_time": 31456.098964214325, "elapsed_training_loop_seconds": 31456.098964214325, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036934930731967, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06122709326631248, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1434, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9650067294751009, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.034993270524899055, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9602960969044414, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9650067294751009, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0054948391151552, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07269751408045111, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1069, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1068, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9561717352415027, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.04382826475849732, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9552772808586762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9552772808586762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9552772808586762, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9561717352415027, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0100427483677161, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06981308012597376, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 586, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 585, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9331210191082803, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9331210191082803, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9331210191082803, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06687898089171974, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9315286624203821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9315286624203821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9315286624203821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9331210191082803, "step_test_time": 9.56096403405536, "step_plot_time": 7.417423966981005, "step_validation_cycle_time": 21.573705212038476} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_40000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_40000.json new file mode 100644 index 0000000000000000000000000000000000000000..a529f003d090f4ea9e169fc628fcfc84896ed2ce --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_40000.json @@ -0,0 +1 @@ +{"epoch": 0.23682792674912226, "train": {"Classification_binary": 3.780867576599121, "Classification": 0.02576938271522522, "Regression_pt": 0.18641695380210876, "Regression_eta": 0.01611465774476528, "Regression_sin_phi": 0.009567196480929852, "Regression_cos_phi": 0.0092622684314847, "Regression_energy": 0.18703091144561768, "Total": 8.370720863342285, "Total_unweighted": 4.215028947219253}, "train_group": {}, "valid": {"Classification_binary": 3.7064614803233047, "Classification": 0.027895377037372996, "Regression_pt": 0.22052915045555602, "Regression_eta": 0.020404676173595673, "Regression_sin_phi": 0.01102621631419405, "Regression_cos_phi": 0.012343998919142054, "Regression_energy": 0.2294616293399892, "Total": 9.191836093334441, "Total_unweighted": 4.228122528563155, "metrics/particle/class_1/efficiency": 0.8893826603889465, "metrics/particle/class_1/pid_accuracy": 0.9797019958496094, "metrics/particle/class_2/efficiency": 0.2220650613307953, "metrics/particle/class_2/pid_accuracy": 0.460721880197525, "metrics/particle/class_3/efficiency": 0.6810512542724609, "metrics/particle/class_3/pid_accuracy": 0.9729011654853821, "metrics/particle/class_4/efficiency": 0.8249496817588806, "metrics/particle/class_4/pid_accuracy": 0.08292683213949203, "metrics/particle/class_5/efficiency": 0.9593495726585388, "metrics/particle/class_5/pid_accuracy": 0.347457617521286, "metrics/particle/count/bias_mean": -0.762666642665863, "metrics/particle/count/mae": 4.383999824523926, "metrics/particle/count/prediction_mean": 68.96533203125, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.08602908253669739, "metrics/particle/event/energy_response_mean": 0.9864575266838074, "metrics/particle/event/met_abs_error": 9.148948669433594, "metrics/particle/event/scalar_pt_relative_abs_error": 0.07240930944681168, "metrics/particle/event/scalar_pt_response_mean": 0.9922160506248474, "metrics/particle/event/vector_pt_closure": 0.08039816468954086, "metrics/particle/matched/delta_eta_abs_mean": 0.02268308401107788, "metrics/particle/matched/delta_phi_abs_mean": 0.01832638308405876, "metrics/particle/matched/delta_r_mean": 0.03239574283361435, "metrics/particle/matched/energy_relative_abs_mean": 0.09865092486143112, "metrics/particle/matched/pid_accuracy": 0.9513312578201294, "metrics/particle/matched/pt_relative_abs_mean": 0.09274397045373917, "metrics/particle/matching/duplicate_fraction": 0.04098677635192871, "metrics/particle/matching/efficiency": 0.7469022274017334, "metrics/particle/matching/f1": 0.7510094046592712, "metrics/particle/matching/purity": 0.7551620006561279, "metrics/particle/matching/target_coverage_dr0p05": 0.6137562990188599, "metrics/particle/matching/target_coverage_dr0p10": 0.7790271043777466, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.6650795340538025, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8125095367431641}, "step_train_time": 4.750081300735474, "step_valid_time": 3.937726461997954, "step_total_time": 3223.329378604889, "elapsed_training_loop_seconds": 3223.329378604889, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9792806988941622, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.16041198241085686, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1352, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1331, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9098250336473755, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9098250336473755, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9098250336473755, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0901749663526245, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.895693135935397, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.895693135935397, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.895693135935397, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9098250336473755, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9791078562159865, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17652089753054057, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1113, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 984, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 972, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8801431127012522, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8840970350404312, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8821156432093231, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.11590296495956873, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8694096601073346, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.8733153638814016, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.871358135365307, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8801431127012522, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9999825625045589, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.17129345871166624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 517, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 512, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.8232484076433121, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.8433931484502447, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.8331990330378727, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.1566068515497553, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.8152866242038217, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.835236541598695, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.8251410153102336, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.8232484076433121, "step_test_time": 9.38439804999507, "step_plot_time": 7.1747824780177325, "step_validation_cycle_time": 20.94403275899822} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_400000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_400000.json new file mode 100644 index 0000000000000000000000000000000000000000..34ade74d86e471aa7344384a1c49ec627b0234ef --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_400000.json @@ -0,0 +1 @@ +{"epoch": 2.3682792674912228, "train": {"Classification_binary": 2.040041208267212, "Classification": 0.0179749708622694, "Regression_pt": 0.16797402501106262, "Regression_eta": 0.020697403699159622, "Regression_sin_phi": 0.012284558266401291, "Regression_cos_phi": 0.010324446484446526, "Regression_energy": 0.19790132343769073, "Total": 6.452852249145508, "Total_unweighted": 2.467197936028242}, "train_group": {}, "valid": {"Classification_binary": 2.119538165153341, "Classification": 0.014770255443897653, "Regression_pt": 0.1151886797965841, "Regression_eta": 0.011490362755795742, "Regression_sin_phi": 0.00644267683333539, "Regression_cos_phi": 0.007078041421606186, "Regression_energy": 0.12040522757996903, "Total": 5.005353887030419, "Total_unweighted": 2.394913408984529, "metrics/particle/class_1/efficiency": 0.9484208822250366, "metrics/particle/class_1/pid_accuracy": 0.9880355000495911, "metrics/particle/class_2/efficiency": 0.46958982944488525, "metrics/particle/class_2/pid_accuracy": 0.8217871189117432, "metrics/particle/class_3/efficiency": 0.8102411031723022, "metrics/particle/class_3/pid_accuracy": 0.9890576004981995, "metrics/particle/class_4/efficiency": 0.8511066436767578, "metrics/particle/class_4/pid_accuracy": 0.5484633445739746, "metrics/particle/class_5/efficiency": 0.9837398529052734, "metrics/particle/class_5/pid_accuracy": 0.7355371713638306, "metrics/particle/count/bias_mean": -0.23199999332427979, "metrics/particle/count/mae": 2.7253334522247314, "metrics/particle/count/prediction_mean": 69.49600219726562, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04501014202833176, "metrics/particle/event/energy_response_mean": 0.9927299618721008, "metrics/particle/event/met_abs_error": 3.888841390609741, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03447501361370087, "metrics/particle/event/scalar_pt_response_mean": 0.9946874380111694, "metrics/particle/event/vector_pt_closure": 0.036735184490680695, "metrics/particle/matched/delta_eta_abs_mean": 0.010453389026224613, "metrics/particle/matched/delta_phi_abs_mean": 0.011293726973235607, "metrics/particle/matched/delta_r_mean": 0.0171922966837883, "metrics/particle/matched/energy_relative_abs_mean": 0.08364900946617126, "metrics/particle/matched/pid_accuracy": 0.9754457473754883, "metrics/particle/matched/pt_relative_abs_mean": 0.07885438203811646, "metrics/particle/matching/duplicate_fraction": 0.024557767435908318, "metrics/particle/matching/efficiency": 0.850409209728241, "metrics/particle/matching/f1": 0.8518263101577759, "metrics/particle/matching/purity": 0.853248119354248, "metrics/particle/matching/target_coverage_dr0p05": 0.8130067586898804, "metrics/particle/matching/target_coverage_dr0p10": 0.8787478804588318, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.759293258190155, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8863393068313599}, "step_train_time": 5.8231282234191895, "step_valid_time": 3.991733768954873, "step_total_time": 32271.521896600723, "elapsed_training_loop_seconds": 32271.521896600723, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0005272804485743, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06098197365581526, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1436, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1430, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9663526244952894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9663526244952894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9663526244952894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.033647375504710635, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9623149394347241, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9623149394347241, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9623149394347241, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9663526244952894, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0001328832831742, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06363590871226965, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1115, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1077, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1075, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9633273703041145, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9659192825112107, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9646215853112405, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.034080717488789235, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9615384615384616, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9641255605381166, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9628302731751007, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9633273703041145, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9958106589769145, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06316459347504544, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 590, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 589, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9394904458598726, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9409888357256778, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9402390438247012, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05901116427432217, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9378980891719745, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9393939393939394, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9386454183266932, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9394904458598726, "step_test_time": 9.676366769999731, "step_plot_time": 7.458897405012976, "step_validation_cycle_time": 21.712478673027363} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_410000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_410000.json new file mode 100644 index 0000000000000000000000000000000000000000..8d58ed10c6b954a5d2c627bd8a555061b7457697 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_410000.json @@ -0,0 +1 @@ +{"epoch": 2.427486249178503, "train": {"Classification_binary": 1.8730647563934326, "Classification": 0.01574045792222023, "Regression_pt": 0.08174815773963928, "Regression_eta": 0.005173135083168745, "Regression_sin_phi": 0.0031627702992409468, "Regression_cos_phi": 0.0047267707996070385, "Regression_energy": 0.08403586596250534, "Total": 3.9453542232513428, "Total_unweighted": 2.067651914199814}, "train_group": {}, "valid": {"Classification_binary": 2.104578545752992, "Classification": 0.014588647700370626, "Regression_pt": 0.11411483237083922, "Regression_eta": 0.011508632213511366, "Regression_sin_phi": 0.006472566026322385, "Regression_cos_phi": 0.007035171731989434, "Regression_energy": 0.1190589336638755, "Total": 4.96234130859375, "Total_unweighted": 2.3773573294599006, "metrics/particle/class_1/efficiency": 0.9484208822250366, "metrics/particle/class_1/pid_accuracy": 0.9868641495704651, "metrics/particle/class_2/efficiency": 0.47949081659317017, "metrics/particle/class_2/pid_accuracy": 0.826941967010498, "metrics/particle/class_3/efficiency": 0.8139955401420593, "metrics/particle/class_3/pid_accuracy": 0.9888302087783813, "metrics/particle/class_4/efficiency": 0.8651911616325378, "metrics/particle/class_4/pid_accuracy": 0.5534883737564087, "metrics/particle/class_5/efficiency": 0.9796748161315918, "metrics/particle/class_5/pid_accuracy": 0.726141095161438, "metrics/particle/count/bias_mean": 0.3400000035762787, "metrics/particle/count/mae": 2.745333433151245, "metrics/particle/count/prediction_mean": 70.06800079345703, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04047773405909538, "metrics/particle/event/energy_response_mean": 1.0038559436798096, "metrics/particle/event/met_abs_error": 3.8103537559509277, "metrics/particle/event/scalar_pt_relative_abs_error": 0.0330982506275177, "metrics/particle/event/scalar_pt_response_mean": 1.006492018699646, "metrics/particle/event/vector_pt_closure": 0.03699309751391411, "metrics/particle/matched/delta_eta_abs_mean": 0.010091189295053482, "metrics/particle/matched/delta_phi_abs_mean": 0.011030726134777069, "metrics/particle/matched/delta_r_mean": 0.016691578552126884, "metrics/particle/matched/energy_relative_abs_mean": 0.08472726494073868, "metrics/particle/matched/pid_accuracy": 0.9747780561447144, "metrics/particle/matched/pt_relative_abs_mean": 0.07939622551202774, "metrics/particle/matching/duplicate_fraction": 0.025403892621397972, "metrics/particle/matching/efficiency": 0.8529141545295715, "metrics/particle/matching/f1": 0.8508397936820984, "metrics/particle/matching/purity": 0.8487754464149475, "metrics/particle/matching/target_coverage_dr0p05": 0.8181122541427612, "metrics/particle/matching/target_coverage_dr0p10": 0.8810998797416687, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7590255737304688, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8880985379219055}, "step_train_time": 7.061070203781128, "step_valid_time": 4.00261311896611, "step_total_time": 33094.5680243969, "elapsed_training_loop_seconds": 33094.5680243969, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0027226108103056, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06016214436153322, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1492, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1439, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.968371467025572, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9644772117962467, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9664204163868368, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.035522788203753354, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9643337819650067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.960455764075067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9623908663532572, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.968371467025572, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0025404473839155, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06574226624215962, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1117, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1078, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1074, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.964221824686941, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9650850492390332, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9646532438478748, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03491495076096687, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.960644007155635, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9615040286481648, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9610738255033557, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.964221824686941, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.000006985031381, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06525022982781359, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 629, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 585, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 584, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9315286624203821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9300476947535771, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9307875894988067, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06995230524642289, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9299363057324841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9284578696343402, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9291964996022275, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9315286624203821, "step_test_time": 10.684329670970328, "step_plot_time": 7.44454322202364, "step_validation_cycle_time": 22.712929757020902} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_420000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_420000.json new file mode 100644 index 0000000000000000000000000000000000000000..3887feace764c833b5273b3ceb2f7931823dfff8 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_420000.json @@ -0,0 +1 @@ +{"epoch": 2.4866932308657836, "train": {"Classification_binary": 2.2536509037017822, "Classification": 0.012037497945129871, "Regression_pt": 0.09156429767608643, "Regression_eta": 0.009928497485816479, "Regression_sin_phi": 0.006177014205604792, "Regression_cos_phi": 0.007252839859575033, "Regression_energy": 0.0985342413187027, "Total": 4.599135875701904, "Total_unweighted": 2.4791452921926975}, "train_group": {}, "valid": {"Classification_binary": 2.094664228723404, "Classification": 0.014630520597417304, "Regression_pt": 0.11304833026642495, "Regression_eta": 0.011672657855013584, "Regression_sin_phi": 0.006422416960939448, "Regression_cos_phi": 0.007037533724561651, "Regression_energy": 0.11798974300952668, "Total": 4.929987481299867, "Total_unweighted": 2.365465431137287, "metrics/particle/class_1/efficiency": 0.9518330693244934, "metrics/particle/class_1/pid_accuracy": 0.9870779514312744, "metrics/particle/class_2/efficiency": 0.48892033100128174, "metrics/particle/class_2/pid_accuracy": 0.8187078237533569, "metrics/particle/class_3/efficiency": 0.813814640045166, "metrics/particle/class_3/pid_accuracy": 0.9892168045043945, "metrics/particle/class_4/efficiency": 0.8651911616325378, "metrics/particle/class_4/pid_accuracy": 0.5488371849060059, "metrics/particle/class_5/efficiency": 0.9796748161315918, "metrics/particle/class_5/pid_accuracy": 0.7095435857772827, "metrics/particle/count/bias_mean": 0.4959999918937683, "metrics/particle/count/mae": 2.7279999256134033, "metrics/particle/count/prediction_mean": 70.2239990234375, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04116816073656082, "metrics/particle/event/energy_response_mean": 0.9963153600692749, "metrics/particle/event/met_abs_error": 3.899739980697632, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03243379667401314, "metrics/particle/event/scalar_pt_response_mean": 1.0023071765899658, "metrics/particle/event/vector_pt_closure": 0.036487895995378494, "metrics/particle/matched/delta_eta_abs_mean": 0.010095319710671902, "metrics/particle/matched/delta_phi_abs_mean": 0.011094278655946255, "metrics/particle/matched/delta_r_mean": 0.016770416870713234, "metrics/particle/matched/energy_relative_abs_mean": 0.08371841907501221, "metrics/particle/matched/pid_accuracy": 0.974422037601471, "metrics/particle/matched/pt_relative_abs_mean": 0.07939954102039337, "metrics/particle/matching/duplicate_fraction": 0.025632262229919434, "metrics/particle/matching/efficiency": 0.8552470803260803, "metrics/particle/matching/f1": 0.8522160053253174, "metrics/particle/matching/purity": 0.8492063283920288, "metrics/particle/matching/target_coverage_dr0p05": 0.8207319974899292, "metrics/particle/matching/target_coverage_dr0p10": 0.8835666179656982, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7611863017082214, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8909668326377869}, "step_train_time": 5.119798183441162, "step_valid_time": 4.019877094018739, "step_total_time": 33919.16790151596, "elapsed_training_loop_seconds": 33919.16790151596, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0008859979553195, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05734442504693482, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1480, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1426, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9629878869448183, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9668918918918918, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9649359406608227, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03310810810810811, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9596231493943472, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9635135135135136, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.961564396493594, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9629878869448183, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0018304212972995, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06373975799380982, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1122, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1084, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1080, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9695885509838998, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.966131907308378, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9678571428571429, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0338680926916221, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9660107334525939, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9625668449197861, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9642857142857143, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9695885509838998, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0021785684015443, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.060951317658461956, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 587, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 587, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9347133757961783, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9362041467304625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9354581673306773, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06379585326953748, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9347133757961783, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9362041467304625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9354581673306773, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9347133757961783, "step_test_time": 9.643581553013064, "step_plot_time": 7.459415541030467, "step_validation_cycle_time": 21.703000229026657} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_430000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_430000.json new file mode 100644 index 0000000000000000000000000000000000000000..e9b5e199bd4d1ef0f22b540f7b687e7af785f1d9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_430000.json @@ -0,0 +1 @@ +{"epoch": 2.545900212553064, "train": {"Classification_binary": 2.355069637298584, "Classification": 0.010245232842862606, "Regression_pt": 0.11340229958295822, "Regression_eta": 0.009307310916483402, "Regression_sin_phi": 0.007567693013697863, "Regression_cos_phi": 0.007312840782105923, "Regression_energy": 0.11556735634803772, "Total": 5.119629859924316, "Total_unweighted": 2.6184723707847297}, "train_group": {}, "valid": {"Classification_binary": 2.089870696372174, "Classification": 0.014430441754929562, "Regression_pt": 0.1120812639277032, "Regression_eta": 0.011408377200999159, "Regression_sin_phi": 0.006426982423092456, "Regression_cos_phi": 0.006950865400598404, "Regression_energy": 0.11686782634004633, "Total": 4.898603723404255, "Total_unweighted": 2.358036453419543, "metrics/particle/class_1/efficiency": 0.9513569474220276, "metrics/particle/class_1/pid_accuracy": 0.9874885082244873, "metrics/particle/class_2/efficiency": 0.4776048958301544, "metrics/particle/class_2/pid_accuracy": 0.8188548684120178, "metrics/particle/class_3/efficiency": 0.8158501982688904, "metrics/particle/class_3/pid_accuracy": 0.9892991781234741, "metrics/particle/class_4/efficiency": 0.863179087638855, "metrics/particle/class_4/pid_accuracy": 0.5617715716362, "metrics/particle/class_5/efficiency": 0.9756097793579102, "metrics/particle/class_5/pid_accuracy": 0.7291666865348816, "metrics/particle/count/bias_mean": 0.13199999928474426, "metrics/particle/count/mae": 2.75600004196167, "metrics/particle/count/prediction_mean": 69.86000061035156, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04204176738858223, "metrics/particle/event/energy_response_mean": 1.007932186126709, "metrics/particle/event/met_abs_error": 3.641899824142456, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03312414512038231, "metrics/particle/event/scalar_pt_response_mean": 1.0109035968780518, "metrics/particle/event/vector_pt_closure": 0.03547009080648422, "metrics/particle/matched/delta_eta_abs_mean": 0.010263150557875633, "metrics/particle/matched/delta_phi_abs_mean": 0.010777024552226067, "metrics/particle/matched/delta_r_mean": 0.01665804721415043, "metrics/particle/matched/energy_relative_abs_mean": 0.08410544693470001, "metrics/particle/matched/pid_accuracy": 0.9751057028770447, "metrics/particle/matched/pt_relative_abs_mean": 0.07903964817523956, "metrics/particle/matching/duplicate_fraction": 0.025441359728574753, "metrics/particle/matching/efficiency": 0.8549219965934753, "metrics/particle/matching/f1": 0.8541135191917419, "metrics/particle/matching/purity": 0.8533065915107727, "metrics/particle/matching/target_coverage_dr0p05": 0.8213630318641663, "metrics/particle/matching/target_coverage_dr0p10": 0.8824192881584167, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7624866366386414, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8897430300712585}, "step_train_time": 4.901632308959961, "step_valid_time": 4.01836586202262, "step_total_time": 34727.28270316124, "elapsed_training_loop_seconds": 34727.28270316124, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0047033426341831, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.060039147215188216, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1489, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1443, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1436, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9710632570659489, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9691067830758898, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9700840336134454, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03089321692411014, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9663526244952894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.964405641370047, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9653781512605042, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9710632570659489, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0069211915640968, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.07140404124119348, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1082, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1078, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9677996422182469, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.966934763181412, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9673670093875727, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03306523681858802, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.964221824686941, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9633601429848079, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9637907912382655, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9677996422182469, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0088106516007909, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06557976169138002, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 589, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 589, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9378980891719745, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9408945686900958, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9393939393939394, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05910543130990415, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9378980891719745, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9408945686900958, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9393939393939394, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9378980891719745, "step_test_time": 9.765688408981077, "step_plot_time": 7.4299891060218215, "step_validation_cycle_time": 21.82816658896627} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_440000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_440000.json new file mode 100644 index 0000000000000000000000000000000000000000..161a8152dac8df96405616d599eb969905962b47 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_440000.json @@ -0,0 +1 @@ +{"epoch": 2.605107194240345, "train": {"Classification_binary": 2.1237478256225586, "Classification": 0.013696457259356976, "Regression_pt": 0.07917407900094986, "Regression_eta": 0.00730829918757081, "Regression_sin_phi": 0.006458770949393511, "Regression_cos_phi": 0.006388566922396421, "Regression_energy": 0.08072126656770706, "Total": 4.155091285705566, "Total_unweighted": 2.3174952655099332}, "train_group": {}, "valid": {"Classification_binary": 2.079950860206117, "Classification": 0.014503946963776933, "Regression_pt": 0.112208254793857, "Regression_eta": 0.011506553660047816, "Regression_sin_phi": 0.006388244476724178, "Regression_cos_phi": 0.006865217330607962, "Regression_energy": 0.11706866609289292, "Total": 4.89264825049867, "Total_unweighted": 2.3484917435240242, "metrics/particle/class_1/efficiency": 0.951475977897644, "metrics/particle/class_1/pid_accuracy": 0.9886577129364014, "metrics/particle/class_2/efficiency": 0.48797735571861267, "metrics/particle/class_2/pid_accuracy": 0.8222222328186035, "metrics/particle/class_3/efficiency": 0.8117790818214417, "metrics/particle/class_3/pid_accuracy": 0.988242506980896, "metrics/particle/class_4/efficiency": 0.8591549396514893, "metrics/particle/class_4/pid_accuracy": 0.5409836173057556, "metrics/particle/class_5/efficiency": 0.9796748161315918, "metrics/particle/class_5/pid_accuracy": 0.7012448310852051, "metrics/particle/count/bias_mean": 0.25066667795181274, "metrics/particle/count/mae": 2.7173333168029785, "metrics/particle/count/prediction_mean": 69.97866821289062, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04036133736371994, "metrics/particle/event/energy_response_mean": 0.998381495475769, "metrics/particle/event/met_abs_error": 3.572963237762451, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03164353594183922, "metrics/particle/event/scalar_pt_response_mean": 1.0038139820098877, "metrics/particle/event/vector_pt_closure": 0.034620754420757294, "metrics/particle/matched/delta_eta_abs_mean": 0.009989012032747269, "metrics/particle/matched/delta_phi_abs_mean": 0.010640156455338001, "metrics/particle/matched/delta_r_mean": 0.016329433768987656, "metrics/particle/matched/energy_relative_abs_mean": 0.08338767290115356, "metrics/particle/matched/pid_accuracy": 0.9749467968940735, "metrics/particle/matched/pt_relative_abs_mean": 0.07960763573646545, "metrics/particle/matching/duplicate_fraction": 0.0251886285841465, "metrics/particle/matching/efficiency": 0.8540806174278259, "metrics/particle/matching/f1": 0.8525481820106506, "metrics/particle/matching/purity": 0.8510212898254395, "metrics/particle/matching/target_coverage_dr0p05": 0.8225677013397217, "metrics/particle/matching/target_coverage_dr0p10": 0.8838343024253845, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7584327459335327, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8884426951408386}, "step_train_time": 4.973574161529541, "step_valid_time": 4.018163775035646, "step_total_time": 35543.38227510452, "elapsed_training_loop_seconds": 35543.38227510452, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.9993355697829293, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05905419213527385, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1435, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9656796769851952, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9663299663299664, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9660047122181084, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03367003367003367, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9622895622895623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9619656681252103, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9656796769851952, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0019583928029587, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06651891128807774, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1122, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1082, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1080, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9677996422182469, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.964349376114082, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9660714285714286, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.035650623885918005, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9660107334525939, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9625668449197861, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9642857142857143, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9677996422182469, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036932011978608, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06094808840693389, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 585, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 584, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9315286624203821, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9330143540669856, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9322709163346613, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.06698564593301436, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9299363057324841, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9314194577352473, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9306772908366534, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9315286624203821, "step_test_time": 9.68598461104557, "step_plot_time": 7.419466213032138, "step_validation_cycle_time": 21.706178003980312} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_450000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_450000.json new file mode 100644 index 0000000000000000000000000000000000000000..ab1a92d18e849f9dde8be49434835c841a81b15f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_450000.json @@ -0,0 +1 @@ +{"epoch": 2.6643141759276254, "train": {"Classification_binary": 2.2923991680145264, "Classification": 0.01822010800242424, "Regression_pt": 0.1677670031785965, "Regression_eta": 0.025865739211440086, "Regression_sin_phi": 0.009875733405351639, "Regression_cos_phi": 0.00882041361182928, "Regression_energy": 0.19864477217197418, "Total": 6.7064714431762695, "Total_unweighted": 2.7215929375961423}, "train_group": {}, "valid": {"Classification_binary": 2.069567254249086, "Classification": 0.014337277158777764, "Regression_pt": 0.11158271546059466, "Regression_eta": 0.01140244463656811, "Regression_sin_phi": 0.006350833684840101, "Regression_cos_phi": 0.006867348513704665, "Regression_energy": 0.11647412117491378, "Total": 4.866111268388464, "Total_unweighted": 2.336581994878485, "metrics/particle/class_1/efficiency": 0.9515950083732605, "metrics/particle/class_1/pid_accuracy": 0.9877418279647827, "metrics/particle/class_2/efficiency": 0.4898632764816284, "metrics/particle/class_2/pid_accuracy": 0.8253127932548523, "metrics/particle/class_3/efficiency": 0.8154430985450745, "metrics/particle/class_3/pid_accuracy": 0.9898485541343689, "metrics/particle/class_4/efficiency": 0.8752515316009521, "metrics/particle/class_4/pid_accuracy": 0.5724138021469116, "metrics/particle/class_5/efficiency": 0.9756097793579102, "metrics/particle/class_5/pid_accuracy": 0.7250000238418579, "metrics/particle/count/bias_mean": 0.3880000114440918, "metrics/particle/count/mae": 2.700000047683716, "metrics/particle/count/prediction_mean": 70.11599731445312, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04104771092534065, "metrics/particle/event/energy_response_mean": 0.9986414313316345, "metrics/particle/event/met_abs_error": 3.708836317062378, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03191491216421127, "metrics/particle/event/scalar_pt_response_mean": 1.0006886720657349, "metrics/particle/event/vector_pt_closure": 0.035107266157865524, "metrics/particle/matched/delta_eta_abs_mean": 0.009804630652070045, "metrics/particle/matched/delta_phi_abs_mean": 0.010681629180908203, "metrics/particle/matched/delta_r_mean": 0.016226312145590782, "metrics/particle/matched/energy_relative_abs_mean": 0.08417896926403046, "metrics/particle/matched/pid_accuracy": 0.9756053686141968, "metrics/particle/matched/pt_relative_abs_mean": 0.07920770347118378, "metrics/particle/matching/duplicate_fraction": 0.0246829055249691, "metrics/particle/matching/efficiency": 0.8559736609458923, "metrics/particle/matching/f1": 0.8535987734794617, "metrics/particle/matching/purity": 0.8512369990348816, "metrics/particle/matching/target_coverage_dr0p05": 0.8260478973388672, "metrics/particle/matching/target_coverage_dr0p10": 0.8841211795806885, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7627543210983276, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8911962509155273}, "step_train_time": 4.854144334793091, "step_valid_time": 4.029268397949636, "step_total_time": 36364.102743148804, "elapsed_training_loop_seconds": 36364.102743148804, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 0.999861939219501, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05747616027458213, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1485, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1442, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1436, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9703903095558546, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.971043771043771, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9707169303264894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.028956228956228958, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9663526244952894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.967003367003367, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9666778862335914, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9703903095558546, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0006574902763106, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06237103543384992, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1122, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1080, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1077, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9660107334525939, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9625668449197861, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9642857142857143, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0374331550802139, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9633273703041145, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9598930481283422, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9616071428571429, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9660107334525939, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.001284817010379, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06202678726043598, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 627, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 594, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 593, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.945859872611465, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9473684210526315, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9466135458167331, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05263157894736842, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9442675159235668, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9457735247208932, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9450199203187251, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.945859872611465, "step_test_time": 9.690171975991689, "step_plot_time": 7.5225614689989015, "step_validation_cycle_time": 21.82368468400091} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_460000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_460000.json new file mode 100644 index 0000000000000000000000000000000000000000..44b60f4d391fc2fad68757aeb87fed0769bb15e0 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_460000.json @@ -0,0 +1 @@ +{"epoch": 2.723521157614906, "train": {"Classification_binary": 1.6962920427322388, "Classification": 0.012958298437297344, "Regression_pt": 0.08520889282226562, "Regression_eta": 0.010600815527141094, "Regression_sin_phi": 0.004548226948827505, "Regression_cos_phi": 0.005128724966198206, "Regression_energy": 0.08655406534671783, "Total": 3.8395538330078125, "Total_unweighted": 1.9012910667806864}, "train_group": {}, "valid": {"Classification_binary": 2.062018374179272, "Classification": 0.014316241791907778, "Regression_pt": 0.11073307280844831, "Regression_eta": 0.011398309088767843, "Regression_sin_phi": 0.006324521404631594, "Regression_cos_phi": 0.006850716281444468, "Regression_energy": 0.1156563251576525, "Total": 4.839743106923205, "Total_unweighted": 2.3272975607121245, "metrics/particle/class_1/efficiency": 0.951475977897644, "metrics/particle/class_1/pid_accuracy": 0.987573504447937, "metrics/particle/class_2/efficiency": 0.487034410238266, "metrics/particle/class_2/pid_accuracy": 0.8305904865264893, "metrics/particle/class_3/efficiency": 0.8123671412467957, "metrics/particle/class_3/pid_accuracy": 0.9894760251045227, "metrics/particle/class_4/efficiency": 0.8692153096199036, "metrics/particle/class_4/pid_accuracy": 0.5625, "metrics/particle/class_5/efficiency": 0.9878048896789551, "metrics/particle/class_5/pid_accuracy": 0.7119341492652893, "metrics/particle/count/bias_mean": -0.0573333315551281, "metrics/particle/count/mae": 2.700000047683716, "metrics/particle/count/prediction_mean": 69.67066955566406, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.040582116693258286, "metrics/particle/event/energy_response_mean": 1.0025798082351685, "metrics/particle/event/met_abs_error": 3.761373281478882, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03234730288386345, "metrics/particle/event/scalar_pt_response_mean": 1.0043258666992188, "metrics/particle/event/vector_pt_closure": 0.03598474711179733, "metrics/particle/matched/delta_eta_abs_mean": 0.009608273394405842, "metrics/particle/matched/delta_phi_abs_mean": 0.010541380383074284, "metrics/particle/matched/delta_r_mean": 0.01594693772494793, "metrics/particle/matched/energy_relative_abs_mean": 0.08397959917783737, "metrics/particle/matched/pid_accuracy": 0.9754705429077148, "metrics/particle/matched/pt_relative_abs_mean": 0.07914450764656067, "metrics/particle/matching/duplicate_fraction": 0.02472585253417492, "metrics/particle/matching/efficiency": 0.854386568069458, "metrics/particle/matching/f1": 0.8547379970550537, "metrics/particle/matching/purity": 0.8550896644592285, "metrics/particle/matching/target_coverage_dr0p05": 0.8258948922157288, "metrics/particle/matching/target_coverage_dr0p10": 0.882648766040802, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7594462037086487, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8882706165313721}, "step_train_time": 4.703143119812012, "step_valid_time": 4.020369401026983, "step_total_time": 37174.92038989067, "elapsed_training_loop_seconds": 37174.92038989067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0038309956334224, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05653308955583092, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1439, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.968371467025572, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9677202420981843, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9680457450386815, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03227975790181574, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9643337819650067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9636852723604573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9640094180961991, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.968371467025572, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0043752311673668, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06737798994000155, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1120, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1081, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1078, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9669051878354203, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9651785714285714, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.966041108132261, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03482142857142857, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.964221824686941, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9625, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9633601429848079, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9669051878354203, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0022508087934292, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.061277196333512696, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 590, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 590, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9394904458598726, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.944, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9417398244213887, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.056, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9394904458598726, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.944, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9417398244213887, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9394904458598726, "step_test_time": 9.829646450001746, "step_plot_time": 7.427515414019581, "step_validation_cycle_time": 21.87001868902007} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_470000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_470000.json new file mode 100644 index 0000000000000000000000000000000000000000..700e04ddd31bf2c6f84f07ee48d55fa2aa40655f --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_470000.json @@ -0,0 +1 @@ +{"epoch": 2.7827281393021863, "train": {"Classification_binary": 2.261049509048462, "Classification": 0.017133688554167747, "Regression_pt": 0.12355998903512955, "Regression_eta": 0.014779617078602314, "Regression_sin_phi": 0.007059763185679913, "Regression_cos_phi": 0.005223498679697514, "Regression_energy": 0.13209159672260284, "Total": 5.398011684417725, "Total_unweighted": 2.560897662304342}, "train_group": {}, "valid": {"Classification_binary": 2.0608655239673372, "Classification": 0.014236693686627328, "Regression_pt": 0.11021438558050926, "Regression_eta": 0.011312155013388775, "Regression_sin_phi": 0.006364684155646791, "Regression_cos_phi": 0.006855165070675789, "Regression_energy": 0.11509760389936731, "Total": 4.825924487824135, "Total_unweighted": 2.324946211373552, "metrics/particle/class_1/efficiency": 0.9508411288261414, "metrics/particle/class_1/pid_accuracy": 0.9886501431465149, "metrics/particle/class_2/efficiency": 0.4905704855918884, "metrics/particle/class_2/pid_accuracy": 0.8284478783607483, "metrics/particle/class_3/efficiency": 0.8146288394927979, "metrics/particle/class_3/pid_accuracy": 0.9895607829093933, "metrics/particle/class_4/efficiency": 0.8692153096199036, "metrics/particle/class_4/pid_accuracy": 0.5601851940155029, "metrics/particle/class_5/efficiency": 0.9756097793579102, "metrics/particle/class_5/pid_accuracy": 0.7333333492279053, "metrics/particle/count/bias_mean": 0.1626666635274887, "metrics/particle/count/mae": 2.687999963760376, "metrics/particle/count/prediction_mean": 69.89066314697266, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.03993683308362961, "metrics/particle/event/energy_response_mean": 0.9997908473014832, "metrics/particle/event/met_abs_error": 3.652494430541992, "metrics/particle/event/scalar_pt_relative_abs_error": 0.031204931437969208, "metrics/particle/event/scalar_pt_response_mean": 1.003886342048645, "metrics/particle/event/vector_pt_closure": 0.03516504913568497, "metrics/particle/matched/delta_eta_abs_mean": 0.009499981999397278, "metrics/particle/matched/delta_phi_abs_mean": 0.010429342277348042, "metrics/particle/matched/delta_r_mean": 0.015784477815032005, "metrics/particle/matched/energy_relative_abs_mean": 0.08364260196685791, "metrics/particle/matched/pid_accuracy": 0.9760547280311584, "metrics/particle/matched/pt_relative_abs_mean": 0.07885744422674179, "metrics/particle/matching/duplicate_fraction": 0.024896027520298958, "metrics/particle/matching/efficiency": 0.8552661538124084, "metrics/particle/matching/f1": 0.8542697429656982, "metrics/particle/matching/purity": 0.8532755970954895, "metrics/particle/matching/target_coverage_dr0p05": 0.828151285648346, "metrics/particle/matching/target_coverage_dr0p10": 0.8835666179656982, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7612437009811401, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8891502022743225}, "step_train_time": 4.788874626159668, "step_valid_time": 4.009587488952093, "step_total_time": 37988.99235057831, "elapsed_training_loop_seconds": 37988.99235057831, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.00384868422573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05687758107863339, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1484, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1429, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9643337819650067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.965633423180593, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9649831649831649, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03436657681940701, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9616419919246298, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9629380053908356, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9622895622895623, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9643337819650067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0027489136964123, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.064607783387165, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1121, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1085, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1082, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9704830053667263, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.967885816235504, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9691826708351943, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.032114183764495985, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9677996422182469, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9652096342551294, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9665029030817329, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9704830053667263, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0016719194104569, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06300558885250274, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 625, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 591, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 591, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9410828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9456, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9433359936153233, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0544, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9410828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9456, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9433359936153233, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9410828025477707, "step_test_time": 9.71693018998485, "step_plot_time": 7.507042277022265, "step_validation_cycle_time": 21.819204225030262} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_480000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_480000.json new file mode 100644 index 0000000000000000000000000000000000000000..53fbd17f2add7d5437f35bb55211013148f6eca4 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_480000.json @@ -0,0 +1 @@ +{"epoch": 2.841935120989467, "train": {"Classification_binary": 2.3156161308288574, "Classification": 0.014056088402867317, "Regression_pt": 0.10443601757287979, "Regression_eta": 0.009092671796679497, "Regression_sin_phi": 0.00887958612293005, "Regression_cos_phi": 0.007241891231387854, "Regression_energy": 0.10212895274162292, "Total": 4.899737358093262, "Total_unweighted": 2.561451338697225}, "train_group": {}, "valid": {"Classification_binary": 2.0553668407683676, "Classification": 0.0142324021522035, "Regression_pt": 0.11002462468248733, "Regression_eta": 0.011289675185020933, "Regression_sin_phi": 0.006350613654928005, "Regression_cos_phi": 0.006795961172022718, "Regression_energy": 0.11487823851565097, "Total": 4.815403715093085, "Total_unweighted": 2.3189383561306807, "metrics/particle/class_1/efficiency": 0.9505634307861328, "metrics/particle/class_1/pid_accuracy": 0.9883129000663757, "metrics/particle/class_2/efficiency": 0.4853842556476593, "metrics/particle/class_2/pid_accuracy": 0.8271005153656006, "metrics/particle/class_3/efficiency": 0.814312219619751, "metrics/particle/class_3/pid_accuracy": 0.9899455904960632, "metrics/particle/class_4/efficiency": 0.8732394576072693, "metrics/particle/class_4/pid_accuracy": 0.5760368704795837, "metrics/particle/class_5/efficiency": 0.9796748161315918, "metrics/particle/class_5/pid_accuracy": 0.7302904725074768, "metrics/particle/count/bias_mean": 0.10400000214576721, "metrics/particle/count/mae": 2.696000099182129, "metrics/particle/count/prediction_mean": 69.83200073242188, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.04242274910211563, "metrics/particle/event/energy_response_mean": 1.0039656162261963, "metrics/particle/event/met_abs_error": 3.6836166381835938, "metrics/particle/event/scalar_pt_relative_abs_error": 0.03179327771067619, "metrics/particle/event/scalar_pt_response_mean": 1.0051342248916626, "metrics/particle/event/vector_pt_closure": 0.035403087735176086, "metrics/particle/matched/delta_eta_abs_mean": 0.009336763061583042, "metrics/particle/matched/delta_phi_abs_mean": 0.01036055851727724, "metrics/particle/matched/delta_r_mean": 0.01561244297772646, "metrics/particle/matched/energy_relative_abs_mean": 0.08350279927253723, "metrics/particle/matched/pid_accuracy": 0.9761489033699036, "metrics/particle/matched/pt_relative_abs_mean": 0.07871057838201523, "metrics/particle/matching/duplicate_fraction": 0.02510787732899189, "metrics/particle/matching/efficiency": 0.8546351790428162, "metrics/particle/matching/f1": 0.8539983034133911, "metrics/particle/matching/purity": 0.8533623814582825, "metrics/particle/matching/target_coverage_dr0p05": 0.8282277584075928, "metrics/particle/matching/target_coverage_dr0p10": 0.882973849773407, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7612054347991943, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8887869119644165}, "step_train_time": 4.8326098918914795, "step_valid_time": 4.023123400984332, "step_total_time": 38807.85012245178, "elapsed_training_loop_seconds": 38807.85012245178, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002942695414422, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05805601389550885, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1433, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1428, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9643337819650067, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9636852723604573, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9640094180961991, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.0363147276395427, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9609690444145357, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9603227975790182, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9606458123107972, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9643337819650067, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0036396838940402, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06411737218412417, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1119, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1083, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1080, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9686940966010733, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.967828418230563, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9682610639248994, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.032171581769437, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9660107334525939, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9651474530831099, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.965578900312919, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9686940966010733, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002838943587252, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06068323821857324, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 626, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 589, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 588, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9378980891719745, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9408945686900958, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9393939393939394, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05910543130990415, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9363057324840764, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.939297124600639, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.937799043062201, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9378980891719745, "step_test_time": 10.89751687599346, "step_plot_time": 7.506358141021337, "step_validation_cycle_time": 23.04825204599183} \ No newline at end of file diff --git a/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_490000.json b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_490000.json new file mode 100644 index 0000000000000000000000000000000000000000..e8ed4211a940b6219c16551bfe856d7f8ceb77b9 --- /dev/null +++ b/fm/v3.2.0+c6879962/pf_fm_experts/cld_seed12345_20261114_035543_450233/history/step_490000.json @@ -0,0 +1 @@ +{"epoch": 2.9011421026767477, "train": {"Classification_binary": 1.5803784132003784, "Classification": 0.013794144615530968, "Regression_pt": 0.06499847024679184, "Regression_eta": 0.006648817099630833, "Regression_sin_phi": 0.003919987939298153, "Regression_cos_phi": 0.003187416587024927, "Regression_energy": 0.06370905786752701, "Total": 3.234893321990967, "Total_unweighted": 1.7366363075561821}, "train_group": {}, "valid": {"Classification_binary": 2.052894429957613, "Classification": 0.014223735383216371, "Regression_pt": 0.1101383959993403, "Regression_eta": 0.011289000511169434, "Regression_sin_phi": 0.006323251318424306, "Regression_cos_phi": 0.006782844345620338, "Regression_energy": 0.11492287859003594, "Total": 4.814358650369847, "Total_unweighted": 2.3165745361054197, "metrics/particle/class_1/efficiency": 0.9510791897773743, "metrics/particle/class_1/pid_accuracy": 0.988486111164093, "metrics/particle/class_2/efficiency": 0.48656293749809265, "metrics/particle/class_2/pid_accuracy": 0.8260658979415894, "metrics/particle/class_3/efficiency": 0.8153978586196899, "metrics/particle/class_3/pid_accuracy": 0.9899589419364929, "metrics/particle/class_4/efficiency": 0.863179087638855, "metrics/particle/class_4/pid_accuracy": 0.5641025900840759, "metrics/particle/class_5/efficiency": 0.9756097793579102, "metrics/particle/class_5/pid_accuracy": 0.737500011920929, "metrics/particle/count/bias_mean": 0.18799999356269836, "metrics/particle/count/mae": 2.694666624069214, "metrics/particle/count/prediction_mean": 69.91600036621094, "metrics/particle/count/target_mean": 69.72799682617188, "metrics/particle/event/energy_relative_abs_error": 0.03980611264705658, "metrics/particle/event/energy_response_mean": 1.0010998249053955, "metrics/particle/event/met_abs_error": 3.6056911945343018, "metrics/particle/event/scalar_pt_relative_abs_error": 0.031898804008960724, "metrics/particle/event/scalar_pt_response_mean": 1.0046191215515137, "metrics/particle/event/vector_pt_closure": 0.034760214388370514, "metrics/particle/matched/delta_eta_abs_mean": 0.00929578859359026, "metrics/particle/matched/delta_phi_abs_mean": 0.010301405563950539, "metrics/particle/matched/delta_r_mean": 0.015540537424385548, "metrics/particle/matched/energy_relative_abs_mean": 0.0834014043211937, "metrics/particle/matched/pid_accuracy": 0.9761680960655212, "metrics/particle/matched/pt_relative_abs_mean": 0.07861074060201645, "metrics/particle/matching/duplicate_fraction": 0.024620020762085915, "metrics/particle/matching/efficiency": 0.8553235530853271, "metrics/particle/matching/f1": 0.8541720509529114, "metrics/particle/matching/purity": 0.8530236482620239, "metrics/particle/matching/target_coverage_dr0p05": 0.8292603492736816, "metrics/particle/matching/target_coverage_dr0p10": 0.8837578296661377, "metrics/particle/matching/target_coverage_rel_pt0p20": 0.7623718976974487, "metrics/particle/matching/target_coverage_rel_pt0p50": 0.8901254534721375}, "step_train_time": 5.029864072799683, "step_valid_time": 4.039989294018596, "step_total_time": 39629.88202357292, "elapsed_training_loop_seconds": 39629.88202357292, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.002448490047977, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.05806876638694036, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1486, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1487, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1436, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1431, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9663526244952894, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9657027572293208, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9660275815674403, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03429724277067922, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9629878869448183, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9623402824478816, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9626639757820383, "step/cld_edm_ttbar_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9663526244952894, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0020816763588114, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.062358252736049335, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 1118, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 1084, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 1081, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9695885509838998, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9695885509838998, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9695885509838998, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.03041144901610018, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9669051878354203, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9669051878354203, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9669051878354203, "step/cld_edm_ww_fullhad_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9695885509838998, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/med": 1.0028291183788371, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/iqr": 0.06258539232764182, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_reference_jets": 628, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_candidate_jets": 624, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_angular_matches": 592, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/num_response_qualified_matches": 591, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_rel_pt_cut": 0.5, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_recall": 0.9426751592356688, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_precision": 0.9487179487179487, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_f1": 0.9456869009584664, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/angular_fake_rate": 0.05128205128205128, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_recall": 0.9410828025477707, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_precision": 0.9471153846153846, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/response_qualified_f1": 0.9440894568690096, "step/cld_edm_qq_pf/jet_ratio/jet_ratio_target_to_pred_pt/match_frac": 0.9426751592356688, "step_test_time": 9.741082769993227, "step_plot_time": 7.507741265988443, "step_validation_cycle_time": 21.88573496398749} \ No newline at end of file