{"timestamp": 1774104231.3426318, "event": "train_step", "step": 10, "epoch": 1, "metrics": {"train/step_loss": 1.6544453685444998, "train/step_real_loss": 1.3328416794538498, "train/lr": 0.00018181818181818183, "train/step_canary_loss": 9.887499809265137, "perf/step_duration_sec": 16.81118968501687, "perf/samples_per_sec": 7.911397259322907, "perf/tokens_per_sec": 6461.232193270026, "perf/logical_batch_size": 133.0, "perf/logical_token_count": 108621.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.347414329116491, "system/cuda_memory_allocated_gb": 16.632154941558838, "system/cuda_max_memory_allocated_gb": 131.87122297286987}}
{"timestamp": 1774104391.3749218, "event": "train_step", "step": 20, "epoch": 1, "metrics": {"train/step_loss": 1.8121112935683306, "train/step_real_loss": 1.3062276244163513, "train/lr": 0.00019897180218885507, "train/step_canary_loss": 9.90625, "perf/step_duration_sec": 16.703223552554846, "perf/samples_per_sec": 8.142140921007915, "perf/tokens_per_sec": 6130.672901419503, "perf/logical_batch_size": 136.0, "perf/logical_token_count": 102402.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.9091472084164676, "system/cuda_memory_allocated_gb": 16.91627025604248, "system/cuda_max_memory_allocated_gb": 131.87122440338135}}
{"timestamp": 1774104551.8943386, "event": "train_step", "step": 30, "epoch": 1, "metrics": {"train/step_loss": 1.5263234859204475, "train/step_real_loss": 1.3384638726711273, "train/lr": 0.00019544467510209388, "train/step_canary_loss": 9.541666984558105, "perf/step_duration_sec": 16.59933592379093, "perf/samples_per_sec": 7.891881976570206, "perf/tokens_per_sec": 6372.1826274025725, "perf/logical_batch_size": 131.0, "perf/logical_token_count": 105774.0, "perf/physical_batches": 9.0, "privacy/epsilon": 3.362310066272621, "system/cuda_memory_allocated_gb": 16.44274616241455, "system/cuda_max_memory_allocated_gb": 131.87122440338135}}
{"timestamp": 1774104714.5782459, "event": "train_step", "step": 40, "epoch": 1, "metrics": {"train/step_loss": 1.687370542270034, "train/step_real_loss": 1.179063692688942, "train/lr": 0.00018949541262593762, "train/step_canary_loss": 12.53125, "perf/step_duration_sec": 16.881952920928597, "perf/samples_per_sec": 7.937470304983488, "perf/tokens_per_sec": 6684.120049885386, "perf/logical_batch_size": 134.0, "perf/logical_token_count": 112841.0, "perf/physical_batches": 9.0, "privacy/epsilon": 3.7564957912533217, "system/cuda_memory_allocated_gb": 16.72686004638672, "system/cuda_max_memory_allocated_gb": 131.8712272644043}}
{"timestamp": 1774104875.8600225, "event": "train_step", "step": 50, "epoch": 1, "metrics": {"train/step_loss": 1.5803235182121618, "train/step_real_loss": 1.2159246057271957, "train/lr": 0.00018127499143005268, "train/step_canary_loss": 9.354166984558105, "perf/step_duration_sec": 16.994498278945684, "perf/samples_per_sec": 7.884904738024027, "perf/tokens_per_sec": 6474.683641371163, "perf/logical_batch_size": 134.0, "perf/logical_token_count": 110034.0, "perf/physical_batches": 9.0, "privacy/epsilon": 4.113499817895439, "system/cuda_memory_allocated_gb": 16.72686004638672, "system/cuda_max_memory_allocated_gb": 131.8712272644043}}
{"timestamp": 1774104902.9951305, "event": "eval_step", "step": 50, "epoch": 1, "metrics": {"eval/loss": 0.8406856013783094, "eval/duration_sec": 27.13257823884487}}
{"timestamp": 1774105063.2638607, "event": "train_step", "step": 60, "epoch": 1, "metrics": {"train/step_loss": 1.5876187704559555, "train/step_real_loss": 1.2753675431013107, "train/lr": 0.0001709920242324663, "train/step_canary_loss": 9.581250190734863, "perf/step_duration_sec": 15.612151749432087, "perf/samples_per_sec": 8.519005075955533, "perf/tokens_per_sec": 6636.881428197056, "perf/logical_batch_size": 133.0, "perf/logical_token_count": 103616.0, "perf/physical_batches": 9.0, "privacy/epsilon": 4.44246859480526, "system/cuda_memory_allocated_gb": 16.632154941558838, "system/cuda_max_memory_allocated_gb": 131.87123012542725}}
{"timestamp": 1774105222.4852285, "event": "train_step", "step": 70, "epoch": 1, "metrics": {"train/step_loss": 1.517450578284986, "train/step_real_loss": 1.238699033856392, "train/lr": 0.00015890746575622231, "train/step_canary_loss": 10.4375, "perf/step_duration_sec": 15.690655367448926, "perf/samples_per_sec": 8.41265051769863, "perf/tokens_per_sec": 6816.286349764424, "perf/logical_batch_size": 132.0, "perf/logical_token_count": 106952.0, "perf/physical_batches": 9.0, "privacy/epsilon": 4.751375515961613, "system/cuda_memory_allocated_gb": 16.53745126724243, "system/cuda_max_memory_allocated_gb": 131.87123012542725}}
{"timestamp": 1774105383.1644452, "event": "train_step", "step": 80, "epoch": 1, "metrics": {"train/step_loss": 1.6151943637016124, "train/step_real_loss": 1.2959636598825455, "train/lr": 0.00014532799038330385, "train/step_canary_loss": 9.787500381469727, "perf/step_duration_sec": 15.639920819550753, "perf/samples_per_sec": 8.503879369628441, "perf/tokens_per_sec": 6513.715841364871, "perf/logical_batch_size": 133.0, "perf/logical_token_count": 101874.0, "perf/physical_batches": 9.0, "privacy/epsilon": 5.041931555886533, "system/cuda_memory_allocated_gb": 16.632154941558838, "system/cuda_max_memory_allocated_gb": 131.87123012542725}}
{"timestamp": 1774105544.5018637, "event": "train_step", "step": 90, "epoch": 1, "metrics": {"train/step_loss": 1.8346073206733255, "train/step_real_loss": 1.1992702782154083, "train/lr": 0.00013059820956358998, "train/step_canary_loss": 12.0, "perf/step_duration_sec": 15.920344103127718, "perf/samples_per_sec": 8.542528925193356, "perf/tokens_per_sec": 7012.285618755411, "perf/logical_batch_size": 136.0, "perf/logical_token_count": 111638.0, "perf/physical_batches": 9.0, "privacy/epsilon": 5.3191578546045415, "system/cuda_memory_allocated_gb": 16.91627025604248, "system/cuda_max_memory_allocated_gb": 131.87123012542725}}
{"timestamp": 1774105706.236452, "event": "train_step", "step": 100, "epoch": 1, "metrics": {"train/step_loss": 1.8485602641451186, "train/step_real_loss": 1.201963409781456, "train/lr": 0.00011509192648058249, "train/step_canary_loss": 10.125, "perf/step_duration_sec": 16.586500560864806, "perf/samples_per_sec": 8.320019011460776, "perf/tokens_per_sec": 6629.547902313321, "perf/logical_batch_size": 138.0, "perf/logical_token_count": 109961.0, "perf/physical_batches": 9.0, "privacy/epsilon": 5.584672549637297, "system/cuda_memory_allocated_gb": 17.105679035186768, "system/cuda_max_memory_allocated_gb": 131.87123012542725}}
{"timestamp": 1774105733.3532517, "event": "eval_step", "step": 100, "epoch": 1, "metrics": {"eval/loss": 0.8407963089559269, "eval/duration_sec": 27.114327052608132}}
{"timestamp": 1774105818.7189045, "event": "train_epoch", "step": 104, "epoch": 1, "metrics": {"train/epoch_loss": 1.6878267472732027, "train/epoch_real_loss": 1.2598683254025247, "train/epoch_canary_loss": 10.317931891543852, "perf/epoch_duration_sec": 1723.9663305338472, "perf/epoch_samples_per_sec": 32.29819458408895, "perf/epoch_tokens_per_sec": 25523.655665812377, "perf/epoch_samples": 55681.0, "perf/epoch_tokens": 44001923.0, "system/cuda_epoch_peak_memory_gb": 131.87123012542725, "eval/loss": 0.8408018544145014, "eval/duration_sec": 27.110809052363038, "privacy/epsilon": 5.688895252675942}}
{"timestamp": 1774105829.5581117, "event": "audit_epoch", "step": 104, "epoch": 1, "metrics": {"audit/delta": 1e-05, "audit/num_canaries": 500.0, "audit/num_members": 250.0, "audit/paper_guess_fraction": 0.2, "audit/paper_guess_steps": 20.0, "audit/loss/auc": 0.523776, "audit/loss/empirical_epsilon/0.05": 0.1320468671619892, "audit/loss/empirical_epsilon/0.01": 0.0, "audit/loss/empirical_epsilon_details/0.05/epsilon": 0.1320468671619892, "audit/loss/empirical_epsilon_details/0.05/num_guesses": 40.0, "audit/loss/empirical_epsilon_details/0.05/correct_guesses": 27.0, "audit/loss/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/loss/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.01/correct_guesses": 0.0, "audit/embedding/auc": 0.517432, "audit/embedding/empirical_epsilon/0.05": 0.0, "audit/embedding/empirical_epsilon/0.01": 0.0, "audit/embedding/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/correct_guesses": 0.0, "perf/audit_duration_sec": 6.300337256863713}}
{"timestamp": 1774105925.8246696, "event": "train_step", "step": 110, "epoch": 2, "metrics": {"train/step_loss": 1.5704498792949475, "train/step_real_loss": 1.2367760390043259, "train/lr": 9.920264990753837e-05, "train/step_canary_loss": 10.112500190734863, "perf/step_duration_sec": 15.937676103785634, "perf/samples_per_sec": 8.345005829827905, "perf/tokens_per_sec": 6717.039504559372, "perf/logical_batch_size": 133.0, "perf/logical_token_count": 107054.0, "perf/physical_batches": 9.0, "privacy/epsilon": 5.840075127464654, "system/cuda_memory_allocated_gb": 16.632154941558838, "system/cuda_max_memory_allocated_gb": 131.87122297286987}}
{"timestamp": 1774106086.7736225, "event": "train_step", "step": 120, "epoch": 2, "metrics": {"train/step_loss": 1.7577342987060547, "train/step_real_loss": 1.249428629875183, "train/lr": 8.333360798744496e-05, "train/step_canary_loss": 9.890625, "perf/step_duration_sec": 16.08638485521078, "perf/samples_per_sec": 8.454354488227118, "perf/tokens_per_sec": 6210.90449465632, "perf/logical_batch_size": 136.0, "perf/logical_token_count": 99911.0, "perf/physical_batches": 9.0, "privacy/epsilon": 6.086971689386329, "system/cuda_memory_allocated_gb": 16.91627025604248, "system/cuda_max_memory_allocated_gb": 131.87122297286987}}
{"timestamp": 1774106246.2913072, "event": "train_step", "step": 130, "epoch": 2, "metrics": {"train/step_loss": 1.4763632254167036, "train/step_real_loss": 1.1850972324609756, "train/lr": 6.788751536089739e-05, "train/step_canary_loss": 10.796875, "perf/step_duration_sec": 15.515285575762391, "perf/samples_per_sec": 8.507738987815168, "perf/tokens_per_sec": 7127.358337042157, "perf/logical_batch_size": 132.0, "perf/logical_token_count": 110583.0, "perf/physical_batches": 9.0, "privacy/epsilon": 6.327001481453001, "system/cuda_memory_allocated_gb": 16.53745126724243, "system/cuda_max_memory_allocated_gb": 131.87122297286987}}
{"timestamp": 1774106405.4323213, "event": "train_step", "step": 140, "epoch": 2, "metrics": {"train/step_loss": 1.3830110549926757, "train/step_real_loss": 1.2400698214769363, "train/lr": 5.325635332531864e-05, "train/step_canary_loss": 10.53125, "perf/step_duration_sec": 15.885074591264129, "perf/samples_per_sec": 8.183782786357986, "perf/tokens_per_sec": 6390.841882217511, "perf/logical_batch_size": 130.0, "perf/logical_token_count": 101519.0, "perf/physical_batches": 9.0, "privacy/epsilon": 6.557571491578655, "system/cuda_memory_allocated_gb": 16.34804105758667, "system/cuda_max_memory_allocated_gb": 131.87122297286987}}
{"timestamp": 1774106564.9002235, "event": "train_step", "step": 150, "epoch": 2, "metrics": {"train/step_loss": 1.4543004363547756, "train/step_real_loss": 1.2664617747068405, "train/lr": 3.981142237826332e-05, "train/step_canary_loss": 9.46875, "perf/step_duration_sec": 15.667665127664804, "perf/samples_per_sec": 8.3611692573573, "perf/tokens_per_sec": 6706.678955912904, "perf/logical_batch_size": 131.0, "perf/logical_token_count": 105078.0, "perf/physical_batches": 9.0, "privacy/epsilon": 6.783113430476456, "system/cuda_memory_allocated_gb": 16.44274616241455, "system/cuda_max_memory_allocated_gb": 131.87122297286987}}
{"timestamp": 1774106591.983752, "event": "eval_step", "step": 150, "epoch": 2, "metrics": {"eval/loss": 0.8408324812678085, "eval/duration_sec": 27.08070291392505}}
{"timestamp": 1774106753.0569706, "event": "train_step", "step": 160, "epoch": 2, "metrics": {"train/step_loss": 1.5232838437073213, "train/step_real_loss": 1.2058339715003967, "train/lr": 2.789391958515183e-05, "train/step_canary_loss": 9.65000057220459, "perf/step_duration_sec": 15.89048214443028, "perf/samples_per_sec": 8.369790091398666, "perf/tokens_per_sec": 6626.545314542769, "perf/logical_batch_size": 133.0, "perf/logical_token_count": 105299.0, "perf/physical_batches": 9.0, "privacy/epsilon": 7.0043611451158165, "system/cuda_memory_allocated_gb": 16.632154941558838, "system/cuda_max_memory_allocated_gb": 131.87122583389282}}
{"timestamp": 1774106914.1120243, "event": "train_step", "step": 170, "epoch": 2, "metrics": {"train/step_loss": 1.4613170623779297, "train/step_real_loss": 1.2692483216524124, "train/lr": 1.7806279893114875e-05, "train/step_canary_loss": 9.65625, "perf/step_duration_sec": 16.511729583144188, "perf/samples_per_sec": 7.933753962015576, "perf/tokens_per_sec": 6002.641909856578, "perf/logical_batch_size": 131.0, "perf/logical_token_count": 99114.0, "perf/physical_batches": 9.0, "privacy/epsilon": 7.2178133573416465, "system/cuda_memory_allocated_gb": 16.44274616241455, "system/cuda_max_memory_allocated_gb": 131.87122583389282}}
{"timestamp": 1774107075.0082667, "event": "train_step", "step": 180, "epoch": 2, "metrics": {"train/step_loss": 1.596443717159442, "train/step_real_loss": 1.2137574702501297, "train/lr": 9.804501125681243e-06, "train/step_canary_loss": 9.760416984558105, "perf/step_duration_sec": 16.71301623620093, "perf/samples_per_sec": 8.017702975107014, "perf/tokens_per_sec": 6444.079182231532, "perf/logical_batch_size": 134.0, "perf/logical_token_count": 107700.0, "perf/physical_batches": 9.0, "privacy/epsilon": 7.430040084733278, "system/cuda_memory_allocated_gb": 16.72686004638672, "system/cuda_max_memory_allocated_gb": 131.87122583389282}}
{"timestamp": 1774107234.6509619, "event": "train_step", "step": 190, "epoch": 2, "metrics": {"train/step_loss": 1.266300171844719, "train/step_real_loss": 1.2088103294372559, "train/lr": 4.091647429802869e-06, "train/step_canary_loss": 8.625, "perf/step_duration_sec": 15.97930870205164, "perf/samples_per_sec": 8.072939975396885, "perf/tokens_per_sec": 6843.600185655053, "perf/logical_batch_size": 129.0, "perf/logical_token_count": 109356.0, "perf/physical_batches": 9.0, "privacy/epsilon": 7.633877151395027, "system/cuda_memory_allocated_gb": 16.25333595275879, "system/cuda_max_memory_allocated_gb": 131.87122583389282}}
{"timestamp": 1774107394.4373736, "event": "train_step", "step": 200, "epoch": 2, "metrics": {"train/step_loss": 1.4888428630250874, "train/step_real_loss": 1.2116387337446213, "train/lr": 8.126960406835249e-07, "train/step_canary_loss": 10.359375, "perf/step_duration_sec": 15.65337066538632, "perf/samples_per_sec": 8.432688576900974, "perf/tokens_per_sec": 6806.010173615901, "perf/logical_batch_size": 132.0, "perf/logical_token_count": 106537.0, "perf/physical_batches": 9.0, "privacy/epsilon": 7.837365803470095, "system/cuda_memory_allocated_gb": 16.53745126724243, "system/cuda_max_memory_allocated_gb": 131.87122583389282}}
{"timestamp": 1774107421.5244071, "event": "eval_step", "step": 200, "epoch": 2, "metrics": {"eval/loss": 0.8408759900893288, "eval/duration_sec": 27.08325556665659}}
{"timestamp": 1774107571.2318742, "event": "train_epoch", "step": 208, "epoch": 2, "metrics": {"train/epoch_loss": 1.630436830589949, "train/epoch_real_loss": 1.2123663491021923, "train/epoch_canary_loss": 10.257968977715942, "perf/epoch_duration_sec": 1714.5822260156274, "perf/epoch_samples_per_sec": 32.43880588290495, "perf/epoch_tokens_per_sec": 25661.50536987952, "perf/epoch_samples": 55619.0, "perf/epoch_tokens": 43998761.0, "system/cuda_epoch_peak_memory_gb": 131.87122583389282, "eval/loss": 0.8408930404432888, "eval/duration_sec": 27.044511837884784, "privacy/epsilon": 7.995186040237391}}
{"timestamp": 1774107582.0385523, "event": "audit_epoch", "step": 208, "epoch": 2, "metrics": {"audit/delta": 1e-05, "audit/num_canaries": 500.0, "audit/num_members": 250.0, "audit/paper_guess_fraction": 0.2, "audit/paper_guess_steps": 20.0, "audit/loss/auc": 0.51964, "audit/loss/empirical_epsilon/0.05": 0.014548719860613346, "audit/loss/empirical_epsilon/0.01": 0.0, "audit/loss/empirical_epsilon_details/0.05/epsilon": 0.014548719860613346, "audit/loss/empirical_epsilon_details/0.05/num_guesses": 35.0, "audit/loss/empirical_epsilon_details/0.05/correct_guesses": 23.0, "audit/loss/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/loss/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.01/correct_guesses": 0.0, "audit/embedding/auc": 0.523288, "audit/embedding/empirical_epsilon/0.05": 0.0, "audit/embedding/empirical_epsilon/0.01": 0.0, "audit/embedding/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/correct_guesses": 0.0, "perf/audit_duration_sec": 6.254005836322904}}
{"timestamp": 1774107593.1762564, "event": "audit_final", "step": 208, "epoch": 2, "metrics": {"audit/delta": 1e-05, "audit/num_canaries": 500.0, "audit/num_members": 250.0, "audit/paper_guess_fraction": 0.2, "audit/paper_guess_steps": 20.0, "audit/loss/auc": 0.51964, "audit/loss/empirical_epsilon/0.05": 0.014548719860613346, "audit/loss/empirical_epsilon/0.01": 0.0, "audit/loss/empirical_epsilon_details/0.05/epsilon": 0.014548719860613346, "audit/loss/empirical_epsilon_details/0.05/num_guesses": 35.0, "audit/loss/empirical_epsilon_details/0.05/correct_guesses": 23.0, "audit/loss/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/loss/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.01/correct_guesses": 0.0, "audit/embedding/auc": 0.523288, "audit/embedding/empirical_epsilon/0.05": 0.0, "audit/embedding/empirical_epsilon/0.01": 0.0, "audit/embedding/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/correct_guesses": 0.0}}
{"timestamp": 1774107593.6992197, "event": "energy_final", "step": 208, "epoch": null, "metrics": {"energy/codecarbon/duration": 3614.081431824714, "energy/codecarbon/emissions": 0.5462366176545209, "energy/codecarbon/emissions_rate": 0.00015114120363877122, "energy/codecarbon/cpu_power": 179.42423601422766, "energy/codecarbon/gpu_power": 2411.5234893258294, "energy/codecarbon/ram_power": 70.0, "energy/codecarbon/cpu_energy": 0.173804537372367, "energy/codecarbon/gpu_energy": 2.4194655916820693, "energy/codecarbon/ram_energy": 0.06780500924975304, "energy/codecarbon/energy_consumed": 2.6610751383041906, "energy/codecarbon/water_consumed": 0.0, "energy/codecarbon/cpu_count": 224.0, "energy/codecarbon/gpu_count": 4.0, "energy/codecarbon/longitude": -121.9552, "energy/codecarbon/latitude": 37.3541, "energy/codecarbon/ram_total_size": 2015.5625190734863, "energy/codecarbon/cpu_utilization_percent": 3.4103505843071784, "energy/codecarbon/gpu_utilization_percent": 96.21675013912076, "energy/codecarbon/ram_utilization_percent": 3.5672787979966607, "energy/codecarbon/ram_used_gb": 71.43525775724738, "energy/codecarbon/pue": 1.0, "energy/codecarbon/wue": 0.0}}