| {"timestamp": 1773811506.4695234, "event": "train_step", "step": 10, "epoch": 1, "metrics": {"train/step_loss": 1.616579128034187, "train/step_real_loss": 1.2012769132852554, "train/lr": 0.0002, "train/step_canary_loss": 14.90625, "perf/step_duration_sec": 12.063218676950783, "perf/samples_per_sec": 5.471176621054408, "perf/tokens_per_sec": 4265.196659189263, "perf/logical_batch_size": 66.0, "perf/logical_token_count": 51452.0, "perf/physical_batches": 9.0, "privacy/epsilon": 0.7638484077719963, "system/cuda_memory_allocated_gb": 12.68057107925415, "system/cuda_max_memory_allocated_gb": 86.2213044166565}} |
| {"timestamp": 1773811628.0656273, "event": "train_step", "step": 20, "epoch": 1, "metrics": {"train/step_loss": 1.7544667159809786, "train/step_real_loss": 1.0926365107297897, "train/lr": 0.0001984111204336116, "train/step_canary_loss": 12.34375, "perf/step_duration_sec": 11.990010250825435, "perf/samples_per_sec": 5.671387978614834, "perf/tokens_per_sec": 4520.429830013588, "perf/logical_batch_size": 68.0, "perf/logical_token_count": 54200.0, "perf/physical_batches": 9.0, "privacy/epsilon": 1.016371889021752, "system/cuda_memory_allocated_gb": 13.261098384857178, "system/cuda_max_memory_allocated_gb": 86.2213044166565}} |
| {"timestamp": 1773811750.6200142, "event": "train_step", "step": 30, "epoch": 1, "metrics": {"train/step_loss": 1.0279401504632197, "train/step_real_loss": 1.0600632801651955, "train/lr": 0.0001936949724999762, "train/step_canary_loss": 0.0, "perf/step_duration_sec": 12.954613069072366, "perf/samples_per_sec": 4.94032509182328, "perf/tokens_per_sec": 3957.7407466073655, "perf/logical_batch_size": 64.0, "perf/logical_token_count": 51271.0, "perf/physical_batches": 10.0, "privacy/epsilon": 1.2175571913066654, "system/cuda_memory_allocated_gb": 14.42089033126831, "system/cuda_max_memory_allocated_gb": 86.2213044166565}} |
| {"timestamp": 1773811872.468319, "event": "train_step", "step": 40, "epoch": 1, "metrics": {"train/step_loss": 1.4674463272094727, "train/step_real_loss": 1.0572493374347687, "train/lr": 0.00018600142402077006, "train/step_canary_loss": 14.59375, "perf/step_duration_sec": 11.865780014079064, "perf/samples_per_sec": 5.562213349791522, "perf/tokens_per_sec": 4472.188085152074, "perf/logical_batch_size": 66.0, "perf/logical_token_count": 53066.0, "perf/physical_batches": 9.0, "privacy/epsilon": 1.3913063211429064, "system/cuda_memory_allocated_gb": 12.680593967437744, "system/cuda_max_memory_allocated_gb": 86.22135162353516}} |
| {"timestamp": 1773811994.1354816, "event": "train_step", "step": 50, "epoch": 1, "metrics": {"train/step_loss": 1.257238564124474, "train/step_real_loss": 1.081570416688919, "train/lr": 0.00017557495743542585, "train/step_canary_loss": 12.5, "perf/step_duration_sec": 12.282207927200943, "perf/samples_per_sec": 5.292208077347962, "perf/tokens_per_sec": 4310.462769707012, "perf/logical_batch_size": 65.0, "perf/logical_token_count": 52942.0, "perf/physical_batches": 9.0, "privacy/epsilon": 1.5474611608458564, "system/cuda_memory_allocated_gb": 12.39033031463623, "system/cuda_max_memory_allocated_gb": 86.22135162353516}} |
| {"timestamp": 1773812006.9120464, "event": "eval_step", "step": 50, "epoch": 1, "metrics": {"eval/loss": 0.9748443443423662, "eval/duration_sec": 12.77427957393229}} |
| {"timestamp": 1773812128.636093, "event": "train_step", "step": 60, "epoch": 1, "metrics": {"train/step_loss": 2.1636021409715926, "train/step_real_loss": 1.041244499385357, "train/lr": 0.0001627469007380852, "train/step_canary_loss": 14.135416984558105, "perf/step_duration_sec": 12.324101509992033, "perf/samples_per_sec": 5.67992725013227, "perf/tokens_per_sec": 4058.7137293088017, "perf/logical_batch_size": 70.0, "perf/logical_token_count": 50020.0, "perf/physical_batches": 9.0, "privacy/epsilon": 1.6892902398315597, "system/cuda_memory_allocated_gb": 13.841647148132324, "system/cuda_max_memory_allocated_gb": 86.22135162353516}} |
| {"timestamp": 1773812251.0683832, "event": "train_step", "step": 70, "epoch": 1, "metrics": {"train/step_loss": 1.8121652182410746, "train/step_real_loss": 1.0885114818811417, "train/lr": 0.0001479248986720057, "train/step_canary_loss": 13.390625, "perf/step_duration_sec": 12.67668361403048, "perf/samples_per_sec": 5.364178997473597, "perf/tokens_per_sec": 3911.117569040308, "perf/logical_batch_size": 68.0, "perf/logical_token_count": 49580.0, "perf/physical_batches": 9.0, "privacy/epsilon": 1.8217089373472166, "system/cuda_memory_allocated_gb": 13.261121273040771, "system/cuda_max_memory_allocated_gb": 86.22135162353516}} |
| {"timestamp": 1773812373.025167, "event": "train_step", "step": 80, "epoch": 1, "metrics": {"train/step_loss": 1.8635356987223906, "train/step_real_loss": 1.08547542989254, "train/lr": 0.0001315799587615025, "train/step_canary_loss": 14.3125, "perf/step_duration_sec": 12.623636846896261, "perf/samples_per_sec": 5.386720231635859, "perf/tokens_per_sec": 3980.3109523349326, "perf/logical_batch_size": 68.0, "perf/logical_token_count": 50246.0, "perf/physical_batches": 9.0, "privacy/epsilon": 1.946082329590544, "system/cuda_memory_allocated_gb": 13.261121273040771, "system/cuda_max_memory_allocated_gb": 86.22135162353516}} |
| {"timestamp": 1773812495.4775374, "event": "train_step", "step": 90, "epoch": 1, "metrics": {"train/step_loss": 1.5933425817916642, "train/step_real_loss": 1.0586555153131485, "train/lr": 0.00011423148382732853, "train/step_canary_loss": 13.0, "perf/step_duration_sec": 11.945233571808785, "perf/samples_per_sec": 5.608931763219984, "perf/tokens_per_sec": 4434.907001318536, "perf/logical_batch_size": 67.0, "perf/logical_token_count": 52976.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.06391520460725, "system/cuda_memory_allocated_gb": 12.970857620239258, "system/cuda_max_memory_allocated_gb": 86.22135162353516}} |
| {"timestamp": 1773812537.9189491, "event": "train_epoch", "step": 93, "epoch": 1, "metrics": {"train/epoch_loss": 1.6919119858601415, "train/epoch_real_loss": 1.0723268173838774, "train/epoch_canary_loss": 13.472450833857557, "perf/epoch_duration_sec": 1142.3914510426112, "perf/epoch_samples_per_sec": 43.508728951566745, "perf/epoch_tokens_per_sec": 33067.53649593877, "perf/epoch_samples": 49704.0, "perf/epoch_tokens": 37776071.0, "system/cuda_epoch_peak_memory_gb": 86.22135162353516, "eval/loss": 0.9517239639774346, "eval/duration_sec": 12.793823145795614, "privacy/epsilon": 2.0981420051109834}} |
| {"timestamp": 1773812551.2877524, "event": "audit_epoch", "step": 93, "epoch": 1, "metrics": {"audit/delta": 1e-05, "audit/num_canaries": 500.0, "audit/num_members": 250.0, "audit/paper_guess_fraction": 0.2, "audit/paper_guess_steps": 20.0, "audit/loss/auc": 0.505664, "audit/loss/empirical_epsilon/0.05": 0.0, "audit/loss/empirical_epsilon/0.01": 0.0, "audit/loss/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/loss/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/loss/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.01/correct_guesses": 0.0, "audit/embedding/auc": 0.51372, "audit/embedding/empirical_epsilon/0.05": 0.0, "audit/embedding/empirical_epsilon/0.01": 0.0, "audit/embedding/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/correct_guesses": 0.0, "perf/audit_duration_sec": 6.93540578102693}} |
| {"timestamp": 1773812637.181298, "event": "train_step", "step": 100, "epoch": 2, "metrics": {"train/step_loss": 2.226203179695237, "train/step_real_loss": 0.9423502832651138, "train/lr": 9.643076661610196e-05, "train/step_canary_loss": 13.964286804199219, "perf/step_duration_sec": 12.210506019182503, "perf/samples_per_sec": 5.814664837678322, "perf/tokens_per_sec": 4380.080556528868, "perf/logical_batch_size": 71.0, "perf/logical_token_count": 53483.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.176204855129592, "system/cuda_memory_allocated_gb": 14.131910800933838, "system/cuda_max_memory_allocated_gb": 86.22135162353516}} |
| {"timestamp": 1773812649.9187398, "event": "eval_step", "step": 100, "epoch": 2, "metrics": {"eval/loss": 0.9495426886356794, "eval/duration_sec": 12.734526461921632}} |
| {"timestamp": 1773812771.789377, "event": "train_step", "step": 110, "epoch": 2, "metrics": {"train/step_loss": 1.4468027822899097, "train/step_real_loss": 1.0408434942364693, "train/lr": 7.874347104470234e-05, "train/step_canary_loss": 14.4375, "perf/step_duration_sec": 12.185235306154937, "perf/samples_per_sec": 5.416391094775368, "perf/tokens_per_sec": 4356.419770834176, "perf/logical_batch_size": 66.0, "perf/logical_token_count": 53084.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.2837597517001855, "system/cuda_memory_allocated_gb": 12.680593967437744, "system/cuda_max_memory_allocated_gb": 86.22137594223022}} |
| {"timestamp": 1773812893.198121, "event": "train_step", "step": 120, "epoch": 2, "metrics": {"train/step_loss": 1.616717566305132, "train/step_real_loss": 1.0352746099233627, "train/lr": 6.173165676349103e-05, "train/step_canary_loss": 14.020833969116211, "perf/step_duration_sec": 12.073515899013728, "perf/samples_per_sec": 5.549336296105193, "perf/tokens_per_sec": 4502.996513587329, "perf/logical_batch_size": 67.0, "perf/logical_token_count": 54367.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.387262501265619, "system/cuda_memory_allocated_gb": 12.970857620239258, "system/cuda_max_memory_allocated_gb": 86.22137594223022}} |
| {"timestamp": 1773813014.0904934, "event": "train_step", "step": 130, "epoch": 2, "metrics": {"train/step_loss": 1.9267044274703315, "train/step_real_loss": 0.9727360382676125, "train/lr": 4.593591825444028e-05, "train/step_canary_loss": 14.137499809265137, "perf/step_duration_sec": 11.957372829318047, "perf/samples_per_sec": 5.770498334786406, "perf/tokens_per_sec": 4662.479024096767, "perf/logical_batch_size": 69.0, "perf/logical_token_count": 55751.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.4870228609511393, "system/cuda_memory_allocated_gb": 13.55138349533081, "system/cuda_max_memory_allocated_gb": 86.22137594223022}} |
| {"timestamp": 1773813135.2851143, "event": "train_step", "step": 140, "epoch": 2, "metrics": {"train/step_loss": 1.6469186313116728, "train/step_real_loss": 1.0473601296544075, "train/lr": 3.185820604061088e-05, "train/step_canary_loss": 14.4375, "perf/step_duration_sec": 12.27732455311343, "perf/samples_per_sec": 5.4572150235296455, "perf/tokens_per_sec": 4390.940368708353, "perf/logical_batch_size": 67.0, "perf/logical_token_count": 53909.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.5836117621736387, "system/cuda_memory_allocated_gb": 12.970857620239258, "system/cuda_max_memory_allocated_gb": 86.22137594223022}} |
| {"timestamp": 1773813257.0352345, "event": "train_step", "step": 150, "epoch": 2, "metrics": {"train/step_loss": 2.0192771579908286, "train/step_real_loss": 1.04129096865654, "train/lr": 1.994587590756397e-05, "train/step_canary_loss": 14.537500381469727, "perf/step_duration_sec": 11.960256013087928, "perf/samples_per_sec": 5.769107277009316, "perf/tokens_per_sec": 4232.434485065052, "perf/logical_batch_size": 69.0, "perf/logical_token_count": 50621.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.6773301521119843, "system/cuda_memory_allocated_gb": 13.55138349533081, "system/cuda_max_memory_allocated_gb": 86.22137594223022}} |
| {"timestamp": 1773813269.8009667, "event": "eval_step", "step": 150, "epoch": 2, "metrics": {"eval/loss": 0.9415916662949781, "eval/duration_sec": 12.763515894301236}} |
| {"timestamp": 1773813391.1203394, "event": "train_step", "step": 160, "epoch": 2, "metrics": {"train/step_loss": 1.4993842009342078, "train/step_real_loss": 1.1243649572134018, "train/lr": 1.057747301402887e-05, "train/step_canary_loss": 13.5, "perf/step_duration_sec": 12.41904565365985, "perf/samples_per_sec": 5.314418018952207, "perf/tokens_per_sec": 3556.311912500654, "perf/logical_batch_size": 66.0, "perf/logical_token_count": 44166.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.768504028383432, "system/cuda_memory_allocated_gb": 12.680593967437744, "system/cuda_max_memory_allocated_gb": 86.22137594223022}} |
| {"timestamp": 1773813513.5407782, "event": "train_step", "step": 170, "epoch": 2, "metrics": {"train/step_loss": 1.6632740604343699, "train/step_real_loss": 1.0752243772149086, "train/lr": 4.050702638550275e-06, "train/step_canary_loss": 14.208333969116211, "perf/step_duration_sec": 12.597235643770546, "perf/samples_per_sec": 5.318627188904904, "perf/tokens_per_sec": 3936.101649771056, "perf/logical_batch_size": 67.0, "perf/logical_token_count": 49584.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.8571453531213207, "system/cuda_memory_allocated_gb": 12.970857620239258, "system/cuda_max_memory_allocated_gb": 86.22137594223022}} |
| {"timestamp": 1773813635.9173324, "event": "train_step", "step": 180, "epoch": 2, "metrics": {"train/step_loss": 1.7563851160161636, "train/step_real_loss": 1.0712373107671738, "train/lr": 5.729698228102653e-07, "train/step_canary_loss": 12.71875, "perf/step_duration_sec": 12.466372530907393, "perf/samples_per_sec": 5.454674150913608, "perf/tokens_per_sec": 4245.501236930199, "perf/logical_batch_size": 68.0, "perf/logical_token_count": 52926.0, "perf/physical_batches": 9.0, "privacy/epsilon": 2.9437684274367397, "system/cuda_memory_allocated_gb": 13.261121273040771, "system/cuda_max_memory_allocated_gb": 86.22137594223022}} |
| {"timestamp": 1773813716.1885753, "event": "train_epoch", "step": 186, "epoch": 2, "metrics": {"train/epoch_loss": 1.6068291228622247, "train/epoch_real_loss": 1.0283400972633716, "train/epoch_canary_loss": 13.124244338677878, "perf/epoch_duration_sec": 1152.0965769039467, "perf/epoch_samples_per_sec": 42.999867366267054, "perf/epoch_tokens_per_sec": 32782.04514893617, "perf/epoch_samples": 49540.0, "perf/epoch_tokens": 37768082.0, "system/cuda_epoch_peak_memory_gb": 86.22137594223022, "eval/loss": 0.9410304912389854, "eval/duration_sec": 12.763936698902398, "privacy/epsilon": 2.9947529815620726}} |
| {"timestamp": 1773813729.4597642, "event": "audit_epoch", "step": 186, "epoch": 2, "metrics": {"audit/delta": 1e-05, "audit/num_canaries": 500.0, "audit/num_members": 250.0, "audit/paper_guess_fraction": 0.2, "audit/paper_guess_steps": 20.0, "audit/loss/auc": 0.504792, "audit/loss/empirical_epsilon/0.05": 0.0, "audit/loss/empirical_epsilon/0.01": 0.0, "audit/loss/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/loss/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/loss/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.01/correct_guesses": 0.0, "audit/embedding/auc": 0.51504, "audit/embedding/empirical_epsilon/0.05": 0.0, "audit/embedding/empirical_epsilon/0.01": 0.0, "audit/embedding/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/correct_guesses": 0.0, "perf/audit_duration_sec": 7.0606719348579645}} |
| {"timestamp": 1773813742.732494, "event": "audit_final", "step": 186, "epoch": 2, "metrics": {"audit/delta": 1e-05, "audit/num_canaries": 500.0, "audit/num_members": 250.0, "audit/paper_guess_fraction": 0.2, "audit/paper_guess_steps": 20.0, "audit/loss/auc": 0.504792, "audit/loss/empirical_epsilon/0.05": 0.0, "audit/loss/empirical_epsilon/0.01": 0.0, "audit/loss/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/loss/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/loss/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/loss/empirical_epsilon_details/0.01/correct_guesses": 0.0, "audit/embedding/auc": 0.51504, "audit/embedding/empirical_epsilon/0.05": 0.0, "audit/embedding/empirical_epsilon/0.01": 0.0, "audit/embedding/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/correct_guesses": 0.0}} |
| {"timestamp": 1773813743.2978687, "event": "energy_final", "step": 186, "epoch": null, "metrics": {"energy/codecarbon/duration": 2457.6702942838892, "energy/codecarbon/emissions": 0.09496692637153915, "energy/codecarbon/emissions_rate": 3.8641036021965845e-05, "energy/codecarbon/cpu_power": 72.03190690693239, "energy/codecarbon/gpu_power": 3303.243429373142, "energy/codecarbon/ram_power": 54.0, "energy/codecarbon/cpu_energy": 0.04736684855609284, "energy/codecarbon/gpu_energy": 2.25075239893377, "energy/codecarbon/ram_energy": 0.03550715308856684, "energy/codecarbon/energy_consumed": 2.3336264005784284, "energy/codecarbon/water_consumed": 0.0, "energy/codecarbon/cpu_count": 256.0, "energy/codecarbon/gpu_count": 8.0, "energy/codecarbon/longitude": 16.1885, "energy/codecarbon/latitude": 58.594, "energy/codecarbon/ram_total_size": 1511.49019241333, "energy/codecarbon/cpu_utilization_percent": 3.8756856324191915, "energy/codecarbon/gpu_utilization_percent": 94.07749795249795, "energy/codecarbon/ram_utilization_percent": 5.391281211624831, "energy/codecarbon/ram_used_gb": 81.57497969388083, "energy/codecarbon/pue": 1.0, "energy/codecarbon/wue": 0.0}} |
|
|
