| {"timestamp": 1773852387.8640254, "event": "train_step", "step": 10, "epoch": 1, "metrics": {"train/step_loss": 1.6353936302128123, "train/step_real_loss": 1.1510175429284573, "train/lr": 0.0002, "train/step_canary_loss": 11.968750158945719, "perf/step_duration_sec": 14.872664371039718, "perf/samples_per_sec": 8.94258060842008, "perf/tokens_per_sec": 6777.198589666931, "perf/logical_batch_size": 133.0, "perf/logical_token_count": 100795.0, "perf/physical_batches": 18.0, "privacy/epsilon": 2.524123361583601, "system/cuda_memory_allocated_gb": 15.702028751373291, "system/cuda_max_memory_allocated_gb": 54.06287908554077}} |
| {"timestamp": 1773852537.5150325, "event": "train_step", "step": 20, "epoch": 1, "metrics": {"train/step_loss": 1.857343329130298, "train/step_real_loss": 1.114402625709772, "train/lr": 0.00019837448439320027, "train/step_canary_loss": 12.42361111111111, "perf/step_duration_sec": 15.259514638921246, "perf/samples_per_sec": 8.91247219968029, "perf/tokens_per_sec": 7023.880020837742, "perf/logical_batch_size": 136.0, "perf/logical_token_count": 107181.0, "perf/physical_batches": 18.0, "privacy/epsilon": 3.1196418977343887, "system/cuda_memory_allocated_gb": 16.279295921325684, "system/cuda_max_memory_allocated_gb": 54.062880516052246}} |
| {"timestamp": 1773852684.6348226, "event": "train_step", "step": 30, "epoch": 1, "metrics": {"train/step_loss": 1.4355280977306943, "train/step_real_loss": 1.148357093334198, "train/lr": 0.0001935507835925601, "train/step_canary_loss": 10.625000238418579, "perf/step_duration_sec": 14.81383830588311, "perf/samples_per_sec": 8.84308288608599, "perf/tokens_per_sec": 6924.876448749963, "perf/logical_batch_size": 131.0, "perf/logical_token_count": 102584.0, "perf/physical_batches": 18.0, "privacy/epsilon": 3.599257677191776, "system/cuda_memory_allocated_gb": 15.317307472229004, "system/cuda_max_memory_allocated_gb": 54.062880516052246}} |
| {"timestamp": 1773852833.9397993, "event": "train_step", "step": 40, "epoch": 1, "metrics": {"train/step_loss": 1.6915703279000742, "train/step_real_loss": 1.1175741739571095, "train/lr": 0.00018568571761675893, "train/step_canary_loss": 12.1875, "perf/step_duration_sec": 14.987364898901433, "perf/samples_per_sec": 8.940864581860026, "perf/tokens_per_sec": 6826.283385380117, "perf/logical_batch_size": 134.0, "perf/logical_token_count": 102308.0, "perf/physical_batches": 18.0, "privacy/epsilon": 4.0171270793767535, "system/cuda_memory_allocated_gb": 15.894390106201172, "system/cuda_max_memory_allocated_gb": 54.062883377075195}} |
| {"timestamp": 1773852981.6699533, "event": "train_step", "step": 50, "epoch": 1, "metrics": {"train/step_loss": 1.730209876872875, "train/step_real_loss": 1.1695572771131992, "train/lr": 0.00017503498221564025, "train/step_canary_loss": 11.982143129621234, "perf/step_duration_sec": 15.535889676073566, "perf/samples_per_sec": 8.625189982287917, "perf/tokens_per_sec": 6820.401162038881, "perf/logical_batch_size": 134.0, "perf/logical_token_count": 105961.0, "perf/physical_batches": 18.0, "privacy/epsilon": 4.394850417905478, "system/cuda_memory_allocated_gb": 15.894390106201172, "system/cuda_max_memory_allocated_gb": 54.062883377075195}} |
| {"timestamp": 1773853001.9967575, "event": "eval_step", "step": 50, "epoch": 1, "metrics": {"eval/loss": 1.0429061587367738, "eval/duration_sec": 20.30736266821623}} |
| {"timestamp": 1773853155.8696322, "event": "train_step", "step": 60, "epoch": 1, "metrics": {"train/step_loss": 1.5988739917152806, "train/step_real_loss": 1.106154203414917, "train/lr": 0.0001619448361215723, "train/step_canary_loss": 14.21250057220459, "perf/step_duration_sec": 14.15656855283305, "perf/samples_per_sec": 9.3949320771935, "perf/tokens_per_sec": 7349.238596324901, "perf/logical_batch_size": 133.0, "perf/logical_token_count": 104040.0, "perf/physical_batches": 17.0, "privacy/epsilon": 4.743740138692227, "system/cuda_memory_allocated_gb": 15.702028751373291, "system/cuda_max_memory_allocated_gb": 54.062886238098145}} |
| {"timestamp": 1773853298.6477532, "event": "train_step", "step": 70, "epoch": 1, "metrics": {"train/step_loss": 1.5077286886446404, "train/step_real_loss": 1.1227163039147854, "train/lr": 0.00014684084406997903, "train/step_canary_loss": 13.828125, "perf/step_duration_sec": 14.256150827044621, "perf/samples_per_sec": 9.25916129826499, "perf/tokens_per_sec": 6918.066538192309, "perf/logical_batch_size": 132.0, "perf/logical_token_count": 98625.0, "perf/physical_batches": 17.0, "privacy/epsilon": 5.069830876941064, "system/cuda_memory_allocated_gb": 15.509668827056885, "system/cuda_max_memory_allocated_gb": 54.062886238098145}} |
| {"timestamp": 1773853446.331358, "event": "train_step", "step": 80, "epoch": 1, "metrics": {"train/step_loss": 1.611925903119539, "train/step_real_loss": 1.1202042661607265, "train/lr": 0.00013021404155695725, "train/step_canary_loss": 14.199999809265137, "perf/step_duration_sec": 14.11704233684577, "perf/samples_per_sec": 9.421236887054398, "perf/tokens_per_sec": 7214.046509883511, "perf/logical_batch_size": 133.0, "perf/logical_token_count": 101841.0, "perf/physical_batches": 17.0, "privacy/epsilon": 5.377982250509876, "system/cuda_memory_allocated_gb": 15.702028751373291, "system/cuda_max_memory_allocated_gb": 54.062886238098145}} |
| {"timestamp": 1773853593.7571762, "event": "train_step", "step": 90, "epoch": 1, "metrics": {"train/step_loss": 1.8932846375625498, "train/step_real_loss": 1.1377343386411667, "train/lr": 0.00011260497112202895, "train/step_canary_loss": 12.63888888888889, "perf/step_duration_sec": 14.762270654086024, "perf/samples_per_sec": 9.21267487819408, "perf/tokens_per_sec": 6934.841014560594, "perf/logical_batch_size": 136.0, "perf/logical_token_count": 102374.0, "perf/physical_batches": 18.0, "privacy/epsilon": 5.671744621870462, "system/cuda_memory_allocated_gb": 16.279295921325684, "system/cuda_max_memory_allocated_gb": 54.062886238098145}} |
| {"timestamp": 1773853638.1426697, "event": "train_epoch", "step": 92, "epoch": 1, "metrics": {"train/epoch_loss": 1.7500966633389758, "train/epoch_real_loss": 1.133818452948967, "train/epoch_canary_loss": 13.288580806015256, "perf/epoch_duration_sec": 1412.2077000171412, "perf/epoch_samples_per_sec": 34.83623549099956, "perf/epoch_tokens_per_sec": 26418.67269208853, "perf/epoch_samples": 49196.0, "perf/epoch_tokens": 37308653.0, "system/cuda_epoch_peak_memory_gb": 54.062886238098145, "eval/loss": 1.03997564451261, "eval/duration_sec": 20.698376037878916, "privacy/epsilon": 5.729164243937991}} |
| {"timestamp": 1773853645.390464, "event": "audit_epoch", "step": 92, "epoch": 1, "metrics": {"audit/delta": 1e-05, "audit/num_canaries": 500.0, "audit/num_members": 250.0, "audit/paper_guess_fraction": 0.2, "audit/paper_guess_steps": 20.0, "audit/loss/auc": 0.552536, "audit/loss/empirical_epsilon/0.05": 0.3197584319859743, "audit/loss/empirical_epsilon/0.01": 0.030605491250753403, "audit/loss/empirical_epsilon_details/0.05/epsilon": 0.3197584319859743, "audit/loss/empirical_epsilon_details/0.05/num_guesses": 25.0, "audit/loss/empirical_epsilon_details/0.05/correct_guesses": 19.0, "audit/loss/empirical_epsilon_details/0.01/epsilon": 0.030605491250753403, "audit/loss/empirical_epsilon_details/0.01/num_guesses": 25.0, "audit/loss/empirical_epsilon_details/0.01/correct_guesses": 19.0, "audit/embedding/auc": 0.507808, "audit/embedding/empirical_epsilon/0.05": 0.0, "audit/embedding/empirical_epsilon/0.01": 0.0, "audit/embedding/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/correct_guesses": 0.0, "perf/audit_duration_sec": 5.064277199096978}} |
| {"timestamp": 1773853767.2709792, "event": "train_step", "step": 100, "epoch": 2, "metrics": {"train/step_loss": 2.0532480357349785, "train/step_real_loss": 1.1272322572767735, "train/lr": 9.458610914145826e-05, "train/step_canary_loss": 13.90625, "perf/step_duration_sec": 14.751106533920392, "perf/samples_per_sec": 9.355230381033918, "perf/tokens_per_sec": 7110.44963025728, "perf/logical_batch_size": 138.0, "perf/logical_token_count": 104887.0, "perf/physical_batches": 18.0, "privacy/epsilon": 5.953156126903914, "system/cuda_memory_allocated_gb": 15.12495470046997, "system/cuda_max_memory_allocated_gb": 54.06288194656372}} |
| {"timestamp": 1773853787.107858, "event": "eval_step", "step": 100, "epoch": 2, "metrics": {"eval/loss": 1.0396132693662272, "eval/duration_sec": 19.829364891862497}} |
| {"timestamp": 1773853935.771681, "event": "train_step", "step": 110, "epoch": 2, "metrics": {"train/step_loss": 1.647982077812081, "train/step_real_loss": 1.1710320189595222, "train/lr": 7.674325444256899e-05, "train/step_canary_loss": 11.822916666666666, "perf/step_duration_sec": 14.729006989859045, "perf/samples_per_sec": 9.029800861087974, "perf/tokens_per_sec": 6888.787551656326, "perf/logical_batch_size": 133.0, "perf/logical_token_count": 101465.0, "perf/physical_batches": 18.0, "privacy/epsilon": 6.2242325072714415, "system/cuda_memory_allocated_gb": 15.702028751373291, "system/cuda_max_memory_allocated_gb": 54.06288194656372}} |
| {"timestamp": 1773854082.9588912, "event": "train_step", "step": 120, "epoch": 2, "metrics": {"train/step_loss": 1.8470192860512837, "train/step_real_loss": 1.1087237671017647, "train/lr": 5.965648380101916e-05, "train/step_canary_loss": 12.347222222222221, "perf/step_duration_sec": 14.77171956608072, "perf/samples_per_sec": 9.206781877465872, "perf/tokens_per_sec": 6978.266784640144, "perf/logical_batch_size": 136.0, "perf/logical_token_count": 103081.0, "perf/physical_batches": 18.0, "privacy/epsilon": 6.487004247951293, "system/cuda_memory_allocated_gb": 16.279295921325684, "system/cuda_max_memory_allocated_gb": 54.06288194656372}} |
| {"timestamp": 1773854228.6299124, "event": "train_step", "step": 130, "epoch": 2, "metrics": {"train/step_loss": 1.4841852043614243, "train/step_real_loss": 1.1038081794977188, "train/lr": 4.388129346376178e-05, "train/step_canary_loss": 13.65625, "perf/step_duration_sec": 14.138089621905237, "perf/samples_per_sec": 9.336480637064444, "perf/tokens_per_sec": 7087.591229068504, "perf/logical_batch_size": 132.0, "perf/logical_token_count": 100205.0, "perf/physical_batches": 17.0, "privacy/epsilon": 6.738888126040064, "system/cuda_memory_allocated_gb": 15.509668827056885, "system/cuda_max_memory_allocated_gb": 54.06288194656372}} |
| {"timestamp": 1773854374.1609344, "event": "train_step", "step": 140, "epoch": 2, "metrics": {"train/step_loss": 1.2703148426900384, "train/step_real_loss": 1.083779253065586, "train/lr": 2.993053979266577e-05, "train/step_canary_loss": 9.229166666666666, "perf/step_duration_sec": 14.757511151023209, "perf/samples_per_sec": 8.809073472459241, "perf/tokens_per_sec": 6979.293388022188, "perf/logical_batch_size": 130.0, "perf/logical_token_count": 102997.0, "perf/physical_batches": 18.0, "privacy/epsilon": 6.9846017598599595, "system/cuda_memory_allocated_gb": 15.124946117401123, "system/cuda_max_memory_allocated_gb": 54.06288194656372}} |
| {"timestamp": 1773854520.2708924, "event": "train_step", "step": 150, "epoch": 2, "metrics": {"train/step_loss": 1.4772591663680914, "train/step_real_loss": 1.1925464905798435, "train/lr": 1.825776614411082e-05, "train/step_canary_loss": 13.625, "perf/step_duration_sec": 14.719724283087999, "perf/samples_per_sec": 8.899623218521182, "perf/tokens_per_sec": 6637.556391749427, "perf/logical_batch_size": 131.0, "perf/logical_token_count": 97703.0, "perf/physical_batches": 17.0, "privacy/epsilon": 7.225647671872536, "system/cuda_memory_allocated_gb": 15.317307472229004, "system/cuda_max_memory_allocated_gb": 54.06288194656372}} |
| {"timestamp": 1773854538.402299, "event": "eval_step", "step": 150, "epoch": 2, "metrics": {"eval/loss": 1.0383169391712586, "eval/duration_sec": 18.111375170992687}} |
| {"timestamp": 1773854691.1560457, "event": "train_step", "step": 160, "epoch": 2, "metrics": {"train/step_loss": 1.6241072825531462, "train/step_real_loss": 1.152385726571083, "train/lr": 9.242458032904311e-06, "train/step_canary_loss": 11.687500476837158, "perf/step_duration_sec": 14.878075499087572, "perf/samples_per_sec": 8.93932820868912, "perf/tokens_per_sec": 6843.62705420095, "perf/logical_batch_size": 133.0, "perf/logical_token_count": 101820.0, "perf/physical_batches": 18.0, "privacy/epsilon": 7.457279858735506, "system/cuda_memory_allocated_gb": 15.702028751373291, "system/cuda_max_memory_allocated_gb": 54.06288194656372}} |
| {"timestamp": 1773854837.3778715, "event": "train_step", "step": 170, "epoch": 2, "metrics": {"train/step_loss": 1.382974067717108, "train/step_real_loss": 1.0921453274786472, "train/lr": 3.1777059397436692e-06, "train/step_canary_loss": 13.791666984558105, "perf/step_duration_sec": 14.022617863025516, "perf/samples_per_sec": 9.342050199158423, "perf/tokens_per_sec": 7510.50916660128, "perf/logical_batch_size": 131.0, "perf/logical_token_count": 105317.0, "perf/physical_batches": 17.0, "privacy/epsilon": 7.687885302283429, "system/cuda_memory_allocated_gb": 15.317307472229004, "system/cuda_max_memory_allocated_gb": 54.06288194656372}} |
| {"timestamp": 1773854982.5852828, "event": "train_step", "step": 180, "epoch": 2, "metrics": {"train/step_loss": 1.7006171685677987, "train/step_real_loss": 1.1432290449738503, "train/lr": 2.606768482050215e-07, "train/step_canary_loss": 11.892857142857142, "perf/step_duration_sec": 14.94509603898041, "perf/samples_per_sec": 8.966151816655827, "perf/tokens_per_sec": 6624.1795798291805, "perf/logical_batch_size": 134.0, "perf/logical_token_count": 98999.0, "perf/physical_batches": 18.0, "privacy/epsilon": 7.908478086773533, "system/cuda_memory_allocated_gb": 15.894390106201172, "system/cuda_max_memory_allocated_gb": 54.06288194656372}} |
| {"timestamp": 1773855055.0742006, "event": "train_epoch", "step": 184, "epoch": 2, "metrics": {"train/epoch_loss": 1.7304604712060176, "train/epoch_real_loss": 1.1275523616137826, "train/epoch_canary_loss": 13.386261670760845, "perf/epoch_duration_sec": 1389.9474938560743, "perf/epoch_samples_per_sec": 35.36968138531038, "perf/epoch_tokens_per_sec": 26840.30811588557, "perf/epoch_samples": 49162.0, "perf/epoch_tokens": 37306619.0, "system/cuda_epoch_peak_memory_gb": 54.06288194656372, "eval/loss": 1.0382178419983232, "eval/duration_sec": 19.597126913955435, "privacy/epsilon": 7.99669228530272}} |
| {"timestamp": 1773855064.748227, "event": "audit_epoch", "step": 184, "epoch": 2, "metrics": {"audit/delta": 1e-05, "audit/num_canaries": 500.0, "audit/num_members": 250.0, "audit/paper_guess_fraction": 0.2, "audit/paper_guess_steps": 20.0, "audit/loss/auc": 0.562848, "audit/loss/empirical_epsilon/0.05": 0.42648759204894304, "audit/loss/empirical_epsilon/0.01": 0.16005427204072475, "audit/loss/empirical_epsilon_details/0.05/epsilon": 0.42648759204894304, "audit/loss/empirical_epsilon_details/0.05/num_guesses": 30.0, "audit/loss/empirical_epsilon_details/0.05/correct_guesses": 23.0, "audit/loss/empirical_epsilon_details/0.01/epsilon": 0.16005427204072475, "audit/loss/empirical_epsilon_details/0.01/num_guesses": 30.0, "audit/loss/empirical_epsilon_details/0.01/correct_guesses": 23.0, "audit/embedding/auc": 0.50796, "audit/embedding/empirical_epsilon/0.05": 0.0, "audit/embedding/empirical_epsilon/0.01": 0.0, "audit/embedding/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/correct_guesses": 0.0, "perf/audit_duration_sec": 7.245345794828609}} |
| {"timestamp": 1773855072.9264297, "event": "audit_final", "step": 184, "epoch": 2, "metrics": {"audit/delta": 1e-05, "audit/num_canaries": 500.0, "audit/num_members": 250.0, "audit/paper_guess_fraction": 0.2, "audit/paper_guess_steps": 20.0, "audit/loss/auc": 0.563808, "audit/loss/empirical_epsilon/0.05": 0.42648759204894304, "audit/loss/empirical_epsilon/0.01": 0.16005427204072475, "audit/loss/empirical_epsilon_details/0.05/epsilon": 0.42648759204894304, "audit/loss/empirical_epsilon_details/0.05/num_guesses": 30.0, "audit/loss/empirical_epsilon_details/0.05/correct_guesses": 23.0, "audit/loss/empirical_epsilon_details/0.01/epsilon": 0.16005427204072475, "audit/loss/empirical_epsilon_details/0.01/num_guesses": 30.0, "audit/loss/empirical_epsilon_details/0.01/correct_guesses": 23.0, "audit/embedding/auc": 0.50796, "audit/embedding/empirical_epsilon/0.05": 0.0, "audit/embedding/empirical_epsilon/0.01": 0.0, "audit/embedding/empirical_epsilon_details/0.05/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.05/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.05/correct_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/epsilon": 0.0, "audit/embedding/empirical_epsilon_details/0.01/num_guesses": 0.0, "audit/embedding/empirical_epsilon_details/0.01/correct_guesses": 0.0}} |
| {"timestamp": 1773855073.4648237, "event": "energy_final", "step": 184, "epoch": null, "metrics": {"energy/codecarbon/duration": 2940.2213213280775, "energy/codecarbon/emissions": 0.04336494635667721, "energy/codecarbon/emissions_rate": 1.4748871468318434e-05, "energy/codecarbon/cpu_power": 82.44163995833674, "energy/codecarbon/gpu_power": 1409.9170137162785, "energy/codecarbon/ram_power": 38.0, "energy/codecarbon/cpu_energy": 0.06497743055379518, "energy/codecarbon/gpu_energy": 1.1496539994446735, "energy/codecarbon/ram_energy": 0.029949873467856485, "energy/codecarbon/energy_consumed": 1.2445813034663262, "energy/codecarbon/water_consumed": 0.0, "energy/codecarbon/cpu_count": 8.0, "energy/codecarbon/gpu_count": 5.0, "energy/codecarbon/longitude": 8.212, "energy/codecarbon/latitude": 47.4843, "energy/codecarbon/ram_total_size": 128.0, "energy/codecarbon/cpu_utilization_percent": 12.307671326287071, "energy/codecarbon/gpu_utilization_percent": 66.43163995908625, "energy/codecarbon/ram_utilization_percent": 8.260415956358626, "energy/codecarbon/ram_used_gb": 166.47142205691932, "energy/codecarbon/pue": 1.0, "energy/codecarbon/wue": 0.0}} |
|
|
