Pj12 commited on
Commit
5d2f297
·
verified ·
1 Parent(s): 43ff4cc

Update train_nsf_sim_cache_sid_load_pretrain.py

Browse files
train_nsf_sim_cache_sid_load_pretrain.py CHANGED
@@ -320,9 +320,11 @@ def run(rank, n_gpus, hps):
320
  elapsed_time_record.pop(0)
321
  elapsed_time_avg = sum(elapsed_time_record)/len(elapsed_time_record)
322
  time_left = elapsed_time_avg*(hps.total_epoch-epoch)
 
323
  hour, minute, second, millisec = Calculate_format_elapsed_time(elapsed_time)
324
  hour_left, minute_left, second_left, millisec_left = Calculate_format_elapsed_time(time_left)
325
- print(f"-Time Elapsed: {hour}h:{formating_time(minute)}m:{formating_time(second)}s:{millisec}ms || Time left: {hour_left}h:{formating_time(minute_left)}m:{formating_time(second_left)}s:{millisec_left}ms-\n")
 
326
  #
327
  if ((len(Loss_Gen_Per_Epoch) - right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch)) + 1) > overtrain and overtrain != -1):
328
  logger.info("Over Train threshold reached. Training is done.")
 
320
  elapsed_time_record.pop(0)
321
  elapsed_time_avg = sum(elapsed_time_record)/len(elapsed_time_record)
322
  time_left = elapsed_time_avg*(hps.total_epoch-epoch)
323
+ time_ovt = elapsed_time_avg*(overtrain-epoch)
324
  hour, minute, second, millisec = Calculate_format_elapsed_time(elapsed_time)
325
  hour_left, minute_left, second_left, millisec_left = Calculate_format_elapsed_time(time_left)
326
+ hour_ovt, minute_ovt, second_ovt, millisec_ovt = Calculate_format_elapsed_time(time_ovt)
327
+ print(f"-Time Elapsed: {hour}h:{formating_time(minute)}m:{formating_time(second)}s:{millisec}ms || Time left: {hour_left}h:{formating_time(minute_left)}m:{formating_time(second_left)}s:{millisec_left}ms-\n-Time to overtrain: {hour_ovt}h:{formating_time(minute_ovt)}m:{formating_time(second_ovt)}s:{millisec_ovt}ms-")
328
  #
329
  if ((len(Loss_Gen_Per_Epoch) - right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch)) + 1) > overtrain and overtrain != -1):
330
  logger.info("Over Train threshold reached. Training is done.")