profiler_cache.csv

gpu_id,timestamp,power,memory,record_step
-1,2025-09-02 14:21:26.558339,-1.0,-1.0,__init__
-1,2025-09-02 14:21:26.713143,-1.0,-1.0,finetuning_start
0,2025-09-02 14:21:26.712000,120.51,5875.0,__unset__
-1,2025-09-02 14:21:26.738855,-1.0,-1.0,evaluate_perplexity_batch_start
0,2025-09-02 14:21:26.815000,120.53,5875.0,__unset__
0,2025-09-02 14:21:26.915000,120.47,5943.0,__unset__
0,2025-09-02 14:21:27.015000,120.49,5943.0,__unset__
0,2025-09-02 14:21:27.115000,120.52,5951.0,__unset__
-1,2025-09-02 14:21:27.147435,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:27.158060,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:27.178732,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:27.181793,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:27.212307,-1.0,-1.0,evaluate_perplexity_batch_end
0,2025-09-02 14:21:27.216000,128.56,5983.0,__unset__
-1,2025-09-02 14:21:27.224877,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:27.316000,135.05,6101.0,__unset__
-1,2025-09-02 14:21:27.327505,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:27.331338,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:27.416000,139.22,6137.0,__unset__
-1,2025-09-02 14:21:27.490342,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:27.492982,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:27.519000,143.03,6175.0,__unset__
-1,2025-09-02 14:21:27.546920,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:27.549597,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:27.619000,154.96,6191.0,__unset__
-1,2025-09-02 14:21:27.643297,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:27.646559,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:27.719000,158.15,6193.0,__unset__
-1,2025-09-02 14:21:27.722710,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:27.725522,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:27.820000,163.42,6203.0,__unset__
-1,2025-09-02 14:21:27.895190,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:27.897766,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:27.921000,165.88,6223.0,__unset__
0,2025-09-02 14:21:28.055000,169.0,6249.0,__unset__
-1,2025-09-02 14:21:28.114731,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.117402,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:28.157000,171.69,6253.0,__unset__
-1,2025-09-02 14:21:28.176447,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.179389,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:28.232942,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.235677,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:28.263000,171.88,6253.0,__unset__
-1,2025-09-02 14:21:28.290108,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.292729,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:28.358999,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.361544,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:28.364000,172.66,6277.0,__unset__
-1,2025-09-02 14:21:28.420457,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.423299,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:28.464000,180.55,6277.0,__unset__
-1,2025-09-02 14:21:28.476210,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.478553,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:28.533817,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.536742,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:28.564000,186.42,6277.0,__unset__
-1,2025-09-02 14:21:28.590204,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.593197,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:28.649888,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.652579,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:28.664000,185.26,6277.0,__unset__
-1,2025-09-02 14:21:28.704707,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.707557,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:28.760822,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.763458,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:28.764000,191.57,6277.0,__unset__
-1,2025-09-02 14:21:28.816862,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.819385,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:28.865000,198.92,6277.0,__unset__
-1,2025-09-02 14:21:28.876855,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:28.891182,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:28.909125,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:28.913095,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:28.940207,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:28.943192,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:28.960910,-1.0,-1.0,evaluate_perplexity_batch_end
0,2025-09-02 14:21:28.965000,206.91,6277.0,__unset__
-1,2025-09-02 14:21:28.969528,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:29.021848,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.024449,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:29.065000,218.36,6277.0,__unset__
-1,2025-09-02 14:21:29.076750,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.079271,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:29.131971,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.134497,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:29.165000,224.7,6277.0,__unset__
-1,2025-09-02 14:21:29.191779,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.194401,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:29.252660,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.255299,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:29.266000,226.18,6277.0,__unset__
-1,2025-09-02 14:21:29.323750,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.326205,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:29.367000,228.65,6277.0,__unset__
-1,2025-09-02 14:21:29.378701,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.381245,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:29.438290,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.440648,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:29.467000,230.22,6277.0,__unset__
-1,2025-09-02 14:21:29.492928,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.495337,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:29.547803,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.550211,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:29.567000,230.74,6277.0,__unset__
-1,2025-09-02 14:21:29.603535,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.606518,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:29.662321,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.665083,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:29.669000,231.67,6277.0,__unset__
-1,2025-09-02 14:21:29.718388,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.720735,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:29.769000,235.93,6277.0,__unset__
-1,2025-09-02 14:21:29.773927,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.776488,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:29.829728,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.832501,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:29.869000,232.3,6281.0,__unset__
-1,2025-09-02 14:21:29.889705,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.892428,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:29.944782,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:29.947263,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:29.969000,232.73,6281.0,__unset__
-1,2025-09-02 14:21:30.000761,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:30.003160,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:30.055755,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:30.058365,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:30.070000,232.05,6281.0,__unset__
-1,2025-09-02 14:21:30.115931,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:30.129663,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:30.147261,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:30.150260,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:30.168221,-1.0,-1.0,evaluate_perplexity_batch_end
0,2025-09-02 14:21:30.170000,230.9,6281.0,__unset__
-1,2025-09-02 14:21:30.171265,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:30.189921,-1.0,-1.0,evaluate_perplexity_batch_end
0,2025-09-02 14:21:30.271000,233.79,6281.0,__unset__
0,2025-09-02 14:21:30.371000,223.65,6281.0,__unset__
0,2025-09-02 14:21:30.471000,211.21,6281.0,__unset__
0,2025-09-02 14:21:30.571000,200.01,6281.0,__unset__
0,2025-09-02 14:21:30.671000,187.9,6281.0,__unset__
-1,2025-09-02 14:21:30.712763,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:30.769456,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:30.771889,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:30.772000,177.13,6281.0,__unset__
-1,2025-09-02 14:21:30.825717,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:30.828427,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:30.872000,172.65,6281.0,__unset__
-1,2025-09-02 14:21:30.881354,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:30.883804,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:30.940755,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:30.943212,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:30.972000,171.82,6281.0,__unset__
-1,2025-09-02 14:21:30.995522,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:30.998236,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:31.051109,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.053521,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:31.072000,172.84,6281.0,__unset__
-1,2025-09-02 14:21:31.107337,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.109905,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:31.167330,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.169964,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:31.172000,174.01,6281.0,__unset__
-1,2025-09-02 14:21:31.244644,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.251226,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:31.273000,172.91,6281.0,__unset__
-1,2025-09-02 14:21:31.303498,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.305762,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:31.358451,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.361113,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:31.373000,177.43,6281.0,__unset__
-1,2025-09-02 14:21:31.418125,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.420393,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:31.471438,-1.0,-1.0,train_step_end
0,2025-09-02 14:21:31.473000,188.71,6281.0,__unset__
-1,2025-09-02 14:21:31.473661,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:31.526713,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.529022,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:31.573000,199.88,6281.0,__unset__
-1,2025-09-02 14:21:31.580430,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.582708,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:31.638458,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.641264,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:31.673000,208.06,6281.0,__unset__
-1,2025-09-02 14:21:31.692624,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.694993,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:31.747068,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.749438,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:31.775000,219.7,6281.0,__unset__
-1,2025-09-02 14:21:31.801412,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.804210,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:31.860180,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.873700,-1.0,-1.0,evaluate_perplexity_batch_start
0,2025-09-02 14:21:31.876000,222.68,6281.0,__unset__
-1,2025-09-02 14:21:31.891295,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:31.894368,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:31.911899,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:31.914845,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:31.932051,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:31.941682,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:31.976000,222.53,6281.0,__unset__
-1,2025-09-02 14:21:31.993616,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:31.996235,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:32.048713,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.051609,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:32.076000,224.38,6281.0,__unset__
-1,2025-09-02 14:21:32.104462,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.107153,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:32.163782,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.166673,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:32.176000,226.54,6281.0,__unset__
-1,2025-09-02 14:21:32.218711,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.221320,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:32.273960,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.276753,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:32.276000,228.1,6281.0,__unset__
-1,2025-09-02 14:21:32.330528,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.333450,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:32.377000,230.94,6281.0,__unset__
-1,2025-09-02 14:21:32.389871,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.392344,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:32.443716,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.446429,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:32.477000,230.52,6281.0,__unset__
-1,2025-09-02 14:21:32.499190,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.501751,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:32.554440,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.557022,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:32.577000,233.66,6281.0,__unset__
-1,2025-09-02 14:21:32.613462,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.616008,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:32.668124,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.671110,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:32.677000,233.06,6281.0,__unset__
-1,2025-09-02 14:21:32.723068,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.725773,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:32.777000,233.62,6281.0,__unset__
-1,2025-09-02 14:21:32.778068,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.781597,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:32.838857,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.841518,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:32.877000,238.37,6281.0,__unset__
-1,2025-09-02 14:21:32.894851,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.897332,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:32.949556,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:32.951993,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:32.978000,238.18,6281.0,__unset__
-1,2025-09-02 14:21:33.004539,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:33.007225,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:33.063022,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:33.076424,-1.0,-1.0,evaluate_perplexity_batch_start
0,2025-09-02 14:21:33.078000,236.68,6281.0,__unset__
-1,2025-09-02 14:21:33.094101,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:33.097281,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:33.114838,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:33.117891,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:33.135128,-1.0,-1.0,evaluate_perplexity_batch_end
0,2025-09-02 14:21:33.178000,234.6,6281.0,__unset__
0,2025-09-02 14:21:33.278000,233.24,6281.0,__unset__
0,2025-09-02 14:21:33.378000,222.21,6281.0,__unset__
0,2025-09-02 14:21:33.478000,209.53,6281.0,__unset__
-1,2025-09-02 14:21:33.484332,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:33.537325,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:33.539954,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:33.578000,198.91,6281.0,__unset__
-1,2025-09-02 14:21:33.592301,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:33.596078,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:33.648399,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:33.650923,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:33.678000,200.52,6281.0,__unset__
-1,2025-09-02 14:21:33.709399,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:33.713113,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:33.767287,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:33.770674,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:33.779000,203.78,6281.0,__unset__
-1,2025-09-02 14:21:33.823721,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:33.826264,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:33.879425,-1.0,-1.0,train_step_end
0,2025-09-02 14:21:33.879000,198.55,6281.0,__unset__
-1,2025-09-02 14:21:33.882163,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:33.940201,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:33.942661,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:33.979000,198.32,6281.0,__unset__
-1,2025-09-02 14:21:33.994408,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:33.997247,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:34.050682,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.053277,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:34.079000,197.22,6281.0,__unset__
-1,2025-09-02 14:21:34.105615,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.108403,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:34.164763,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.167169,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:34.179000,197.6,6281.0,__unset__
-1,2025-09-02 14:21:34.218342,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.221740,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:34.274172,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.276949,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:34.279000,198.25,6281.0,__unset__
-1,2025-09-02 14:21:34.329394,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.332043,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:34.379000,207.55,6281.0,__unset__
-1,2025-09-02 14:21:34.387864,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.390365,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:34.442155,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.444592,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:34.479000,221.29,6281.0,__unset__
-1,2025-09-02 14:21:34.496805,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.503206,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:34.555836,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.558272,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:34.580000,229.58,6281.0,__unset__
-1,2025-09-02 14:21:34.614366,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.627743,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:34.645441,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:34.648542,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:34.665898,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:34.668844,-1.0,-1.0,evaluate_perplexity_batch_start
0,2025-09-02 14:21:34.680000,229.98,6281.0,__unset__
-1,2025-09-02 14:21:34.686418,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:34.695673,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:34.747497,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.750144,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:34.780000,230.28,6281.0,__unset__
-1,2025-09-02 14:21:34.802051,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.804451,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:34.856958,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.859281,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:34.880000,232.0,6281.0,__unset__
-1,2025-09-02 14:21:34.920123,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:34.925060,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:34.978933,-1.0,-1.0,train_step_end
0,2025-09-02 14:21:34.980000,229.02,6281.0,__unset__
-1,2025-09-02 14:21:34.981474,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:35.034602,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.037198,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:35.080000,228.85,6281.0,__unset__
-1,2025-09-02 14:21:35.089628,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.092199,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:35.148357,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.151121,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:35.180000,227.55,6281.0,__unset__
-1,2025-09-02 14:21:35.202838,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.205226,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:35.257411,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.260042,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:35.280000,230.44,6281.0,__unset__
-1,2025-09-02 14:21:35.312172,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.314581,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:35.371334,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.373813,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:35.381000,229.87,6281.0,__unset__
-1,2025-09-02 14:21:35.425203,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.427742,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:35.479610,-1.0,-1.0,train_step_end
0,2025-09-02 14:21:35.481000,227.57,6281.0,__unset__
-1,2025-09-02 14:21:35.482281,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:35.541747,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.544416,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:35.581000,227.76,6281.0,__unset__
-1,2025-09-02 14:21:35.601555,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.604811,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:35.657168,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.660236,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:35.681000,229.24,6281.0,__unset__
-1,2025-09-02 14:21:35.712768,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.715804,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:35.768855,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.771390,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:35.781000,227.57,6281.0,__unset__
-1,2025-09-02 14:21:35.829320,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:35.842592,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:35.860256,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:35.863414,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:35.881040,-1.0,-1.0,evaluate_perplexity_batch_end
0,2025-09-02 14:21:35.881000,225.09,6281.0,__unset__
-1,2025-09-02 14:21:35.884339,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:35.902019,-1.0,-1.0,evaluate_perplexity_batch_end
0,2025-09-02 14:21:35.981000,228.29,6281.0,__unset__
0,2025-09-02 14:21:36.081000,222.27,6281.0,__unset__
0,2025-09-02 14:21:36.182000,211.75,6281.0,__unset__
-1,2025-09-02 14:21:36.267248,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:36.282000,201.08,6281.0,__unset__
-1,2025-09-02 14:21:36.322872,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.326334,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:36.380585,-1.0,-1.0,train_step_end
0,2025-09-02 14:21:36.382000,192.65,6281.0,__unset__
-1,2025-09-02 14:21:36.383272,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:36.437510,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.440344,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:36.482000,194.7,6281.0,__unset__
-1,2025-09-02 14:21:36.496799,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.499216,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:36.551282,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.554096,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:36.583000,196.14,6281.0,__unset__
-1,2025-09-02 14:21:36.607089,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.610100,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:36.663065,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.665409,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:36.683000,195.87,6281.0,__unset__
-1,2025-09-02 14:21:36.721577,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.724329,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:36.776932,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.779504,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:36.783000,196.26,6281.0,__unset__
-1,2025-09-02 14:21:36.833475,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.838148,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:36.883000,196.98,6281.0,__unset__
-1,2025-09-02 14:21:36.891476,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.894282,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:36.951569,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:36.954114,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:36.983000,195.63,6281.0,__unset__
-1,2025-09-02 14:21:37.006292,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.009045,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:37.061421,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.063878,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:37.083000,204.13,6281.0,__unset__
-1,2025-09-02 14:21:37.116969,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.119817,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:37.175873,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.178558,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:37.183000,217.82,6281.0,__unset__
-1,2025-09-02 14:21:37.230643,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.233287,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:37.283000,226.79,6281.0,__unset__
-1,2025-09-02 14:21:37.285555,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.288489,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:37.342720,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.345560,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:37.384000,231.98,6281.0,__unset__
-1,2025-09-02 14:21:37.401655,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.415202,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:37.433143,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:37.436847,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:37.454331,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:37.457724,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:37.475313,-1.0,-1.0,evaluate_perplexity_batch_end
0,2025-09-02 14:21:37.484000,232.19,6281.0,__unset__
-1,2025-09-02 14:21:37.484740,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:37.538443,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.541231,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:37.584000,231.68,6281.0,__unset__
-1,2025-09-02 14:21:37.593597,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.596190,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:37.650551,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.653487,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:37.685000,231.2,6281.0,__unset__
-1,2025-09-02 14:21:37.710228,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.712623,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:37.764883,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.767653,-1.0,-1.0,train_step_start
0,2025-09-02 14:21:37.785000,231.95,6281.0,__unset__
-1,2025-09-02 14:21:37.820782,-1.0,-1.0,train_step_end
-1,2025-09-02 14:21:37.823129,-1.0,-1.0,train_step_start
-1,2025-09-02 14:21:37.876065,-1.0,-1.0,train_step_end
0,2025-09-02 14:21:37.885000,233.9,6281.0,__unset__
-1,2025-09-02 14:21:37.893059,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:37.910697,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:37.914004,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:37.931565,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:37.934761,-1.0,-1.0,evaluate_perplexity_batch_start
-1,2025-09-02 14:21:37.960588,-1.0,-1.0,evaluate_perplexity_batch_end
-1,2025-09-02 14:21:37.978464,-1.0,-1.0,finetuning_end
0,2025-09-02 14:21:37.985000,230.25,6281.0,__unset__