Update train_nsf_sim_cache_sid_load_pretrain.py
Browse files
train_nsf_sim_cache_sid_load_pretrain.py
CHANGED
|
@@ -320,9 +320,11 @@ def run(rank, n_gpus, hps):
|
|
| 320 |
elapsed_time_record.pop(0)
|
| 321 |
elapsed_time_avg = sum(elapsed_time_record)/len(elapsed_time_record)
|
| 322 |
time_left = elapsed_time_avg*(hps.total_epoch-epoch)
|
|
|
|
| 323 |
hour, minute, second, millisec = Calculate_format_elapsed_time(elapsed_time)
|
| 324 |
hour_left, minute_left, second_left, millisec_left = Calculate_format_elapsed_time(time_left)
|
| 325 |
-
|
|
|
|
| 326 |
#
|
| 327 |
if ((len(Loss_Gen_Per_Epoch) - right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch)) + 1) > overtrain and overtrain != -1):
|
| 328 |
logger.info("Over Train threshold reached. Training is done.")
|
|
|
|
| 320 |
elapsed_time_record.pop(0)
|
| 321 |
elapsed_time_avg = sum(elapsed_time_record)/len(elapsed_time_record)
|
| 322 |
time_left = elapsed_time_avg*(hps.total_epoch-epoch)
|
| 323 |
+
time_ovt = elapsed_time_avg*(overtrain-epoch)
|
| 324 |
hour, minute, second, millisec = Calculate_format_elapsed_time(elapsed_time)
|
| 325 |
hour_left, minute_left, second_left, millisec_left = Calculate_format_elapsed_time(time_left)
|
| 326 |
+
hour_ovt, minute_ovt, second_ovt, millisec_ovt = Calculate_format_elapsed_time(time_ovt)
|
| 327 |
+
print(f"-Time Elapsed: {hour}h:{formating_time(minute)}m:{formating_time(second)}s:{millisec}ms || Time left: {hour_left}h:{formating_time(minute_left)}m:{formating_time(second_left)}s:{millisec_left}ms-\n-Time to overtrain: {hour_ovt}h:{formating_time(minute_ovt)}m:{formating_time(second_ovt)}s:{millisec_ovt}ms-")
|
| 328 |
#
|
| 329 |
if ((len(Loss_Gen_Per_Epoch) - right_index(Loss_Gen_Per_Epoch,min(Loss_Gen_Per_Epoch)) + 1) > overtrain and overtrain != -1):
|
| 330 |
logger.info("Over Train threshold reached. Training is done.")
|