2025-10-15 21:01:53,722 - train - INFO - ConformerModel(
  (subsample): ConvSubsampling(
    (conv): Sequential(
      (0): Conv2d(1, 256, kernel_size=(3, 3), stride=(1, 2), padding=(1, 1))
      (1): ReLU()
      (2): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 2), padding=(1, 1))
      (3): ReLU()
    )
  )
  (pre_proj): LinearProjection(
    (proj): Linear(in_features=65536, out_features=512, bias=True)
  )
  (pos_enc): RelativePositionalEncoding()
  (blocks): ModuleList(
    (0-15): 16 x ConformerBlock(
      (ff1): FeedForwardModule(
        (layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
        (linear1): Linear(in_features=512, out_features=2048, bias=True)
        (swish): Swish()
        (dropout): Dropout(p=0.1, inplace=False)
        (linear2): Linear(in_features=2048, out_features=512, bias=True)
        (dropout2): Dropout(p=0.1, inplace=False)
      )
      (mhsa): MultiHeadSelfAttention(
        (layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
        (attn): MultiheadAttention(
          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
        )
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (conv): ConformerConvModule(
        (layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
        (pw_conv1): Conv1d(512, 1024, kernel_size=(1,), stride=(1,))
        (glu): GLU(dim=1)
        (dw_conv): Conv1d(512, 512, kernel_size=(31,), stride=(1,), padding=(15,), groups=512)
        (bn): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (swish): Swish()
        (pw_conv2): Conv1d(512, 512, kernel_size=(1,), stride=(1,))
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (ff2): FeedForwardModule(
        (layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
        (linear1): Linear(in_features=512, out_features=2048, bias=True)
        (swish): Swish()
        (dropout): Dropout(p=0.1, inplace=False)
        (linear2): Linear(in_features=2048, out_features=512, bias=True)
        (dropout2): Dropout(p=0.1, inplace=False)
      )
      (final_ln): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
    )
  )
  (ctc_head): Linear(in_features=512, out_features=28, bias=True)
)
All parameters: 131720732
Trainable parameters: 131720732
2025-10-15 21:01:53,739 - pyctcdecode.alphabet - INFO - Alphabet determined to be of regular style.
2025-10-15 21:01:53,741 - pyctcdecode.alphabet - INFO - Alphabet determined to be of regular style.
2025-10-15 21:28:28,462 - train - INFO -     epoch          : 1
2025-10-15 21:28:28,464 - train - INFO -     loss           : 2.153041486740112
2025-10-15 21:28:28,465 - train - INFO -     grad_norm      : 1.188998259305954
2025-10-15 21:28:28,466 - train - INFO -     val_loss       : 2.172110017234757
2025-10-15 21:28:28,553 - train - INFO -     val_CER_(Argmax): 0.5358357997251945
2025-10-15 21:28:28,554 - train - INFO -     val_WER_(Argmax): 1.012158696859347
2025-10-15 21:28:28,555 - train - INFO -     test_loss      : 2.1243614394490313
2025-10-15 21:28:28,556 - train - INFO -     test_CER_(Argmax): 0.5229153065123915
2025-10-15 21:28:28,557 - train - INFO -     test_WER_(Argmax): 1.00157116613905
2025-10-15 21:28:31,531 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 21:50:34,559 - train - INFO -     epoch          : 2
2025-10-15 21:50:34,561 - train - INFO -     loss           : 1.4212745833396911
2025-10-15 21:50:34,562 - train - INFO -     grad_norm      : 1.9143898677825928
2025-10-15 21:50:34,563 - train - INFO -     val_loss       : 1.475996158532137
2025-10-15 21:50:34,564 - train - INFO -     val_CER_(Argmax): 0.3734411473874421
2025-10-15 21:50:34,564 - train - INFO -     val_WER_(Argmax): 0.8524215912740724
2025-10-15 21:50:34,565 - train - INFO -     test_loss      : 1.4299900364585039
2025-10-15 21:50:34,654 - train - INFO -     test_CER_(Argmax): 0.36053000087408704
2025-10-15 21:50:34,655 - train - INFO -     test_WER_(Argmax): 0.837277992722096
2025-10-15 21:50:37,890 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 22:11:52,360 - train - INFO -     epoch          : 3
2025-10-15 22:11:52,362 - train - INFO -     loss           : 1.1054761481285096
2025-10-15 22:11:52,363 - train - INFO -     grad_norm      : 2.133445930480957
2025-10-15 22:11:52,364 - train - INFO -     val_loss       : 1.179566823166503
2025-10-15 22:11:52,365 - train - INFO -     val_CER_(Argmax): 0.3000637993717897
2025-10-15 22:11:52,366 - train - INFO -     val_WER_(Argmax): 0.7330474730606038
2025-10-15 22:11:52,367 - train - INFO -     test_loss      : 1.1337390374846574
2025-10-15 22:11:52,368 - train - INFO -     test_CER_(Argmax): 0.2865948095416102
2025-10-15 22:11:52,369 - train - INFO -     test_WER_(Argmax): 0.7135280252571233
2025-10-15 22:11:55,916 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 22:32:39,457 - train - INFO -     epoch          : 4
2025-10-15 22:32:39,459 - train - INFO -     loss           : 0.8954937684535981
2025-10-15 22:32:39,460 - train - INFO -     grad_norm      : 2.265110855102539
2025-10-15 22:32:39,462 - train - INFO -     val_loss       : 1.0523416495182105
2025-10-15 22:32:39,462 - train - INFO -     val_CER_(Argmax): 0.26451346666147674
2025-10-15 22:32:39,464 - train - INFO -     val_WER_(Argmax): 0.6780852042352846
2025-10-15 22:32:39,465 - train - INFO -     test_loss      : 1.0155282700207175
2025-10-15 22:32:39,465 - train - INFO -     test_CER_(Argmax): 0.25441022118607404
2025-10-15 22:32:39,466 - train - INFO -     test_WER_(Argmax): 0.6582085265629509
2025-10-15 22:32:42,588 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 22:52:16,859 - train - INFO -     epoch          : 5
2025-10-15 22:52:16,861 - train - INFO -     loss           : 0.7874286377429962
2025-10-15 22:52:16,862 - train - INFO -     grad_norm      : 2.3151290917396548
2025-10-15 22:52:16,863 - train - INFO -     val_loss       : 0.921425876532786
2025-10-15 22:52:16,864 - train - INFO -     val_CER_(Argmax): 0.22771190898381688
2025-10-15 22:52:16,864 - train - INFO -     val_WER_(Argmax): 0.6050087899838589
2025-10-15 22:52:16,865 - train - INFO -     test_loss      : 0.8859942057510701
2025-10-15 22:52:16,866 - train - INFO -     test_CER_(Argmax): 0.21756451149393294
2025-10-15 22:52:16,867 - train - INFO -     test_WER_(Argmax): 0.5850164614033462
2025-10-15 22:52:19,966 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 22:59:22,179 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ReadTimeoutError("HTTPSConnectionPool(host='www.comet.com', port=443): Read timed out. (read timeout=10)")': /clientlib/batch/logger/experiment/metric
2025-10-15 23:00:32,468 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ReadTimeoutError("HTTPSConnectionPool(host='www.comet.com', port=443): Read timed out. (read timeout=10)")': /clientlib/rest/v2/write/experiment/output
2025-10-15 23:10:58,659 - train - INFO -     epoch          : 6
2025-10-15 23:10:58,663 - train - INFO -     loss           : 0.603953384757042
2025-10-15 23:10:58,668 - train - INFO -     grad_norm      : 2.3825424408912657
2025-10-15 23:10:58,673 - train - INFO -     val_loss       : 0.8432797542690526
2025-10-15 23:10:58,674 - train - INFO -     val_CER_(Argmax): 0.19975002812164636
2025-10-15 23:10:58,678 - train - INFO -     val_WER_(Argmax): 0.5460640429402533
2025-10-15 23:10:58,679 - train - INFO -     test_loss      : 0.8142017166062099
2025-10-15 23:10:58,683 - train - INFO -     test_CER_(Argmax): 0.1905588444625169
2025-10-15 23:10:58,687 - train - INFO -     test_WER_(Argmax): 0.5286493462729215
2025-10-15 23:11:01,982 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 23:15:33,860 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ReadTimeoutError("HTTPSConnectionPool(host='www.comet.com', port=443): Read timed out. (read timeout=10)")': /clientlib/batch/logger/experiment/metric
2025-10-15 23:29:41,360 - train - INFO -     epoch          : 7
2025-10-15 23:29:41,362 - train - INFO -     loss           : 0.5495029705762863
2025-10-15 23:29:41,363 - train - INFO -     grad_norm      : 2.781264133453369
2025-10-15 23:29:41,364 - train - INFO -     val_loss       : 0.7903630255594761
2025-10-15 23:29:41,365 - train - INFO -     val_CER_(Argmax): 0.18955925445322028
2025-10-15 23:29:41,366 - train - INFO -     val_WER_(Argmax): 0.5232188275226236
2025-10-15 23:29:41,367 - train - INFO -     test_loss      : 0.7601223377192893
2025-10-15 23:29:41,368 - train - INFO -     test_CER_(Argmax): 0.1790136937010882
2025-10-15 23:29:41,368 - train - INFO -     test_WER_(Argmax): 0.5031266710478192
2025-10-15 23:29:44,559 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 23:47:30,958 - train - INFO -     epoch          : 8
2025-10-15 23:47:30,960 - train - INFO -     loss           : 0.4621651893854141
2025-10-15 23:47:30,961 - train - INFO -     grad_norm      : 2.654181697368622
2025-10-15 23:47:30,962 - train - INFO -     val_loss       : 0.7660906538808134
2025-10-15 23:47:30,962 - train - INFO -     val_CER_(Argmax): 0.17487329416638675
2025-10-15 23:47:30,963 - train - INFO -     val_WER_(Argmax): 0.4908353581541641
2025-10-15 23:47:30,964 - train - INFO -     test_loss      : 0.7469499325606881
2025-10-15 23:47:30,965 - train - INFO -     test_CER_(Argmax): 0.16683605401739357
2025-10-15 23:47:30,965 - train - INFO -     test_WER_(Argmax): 0.4732129735921914
2025-10-15 23:47:34,214 - train - INFO - Saving current best: model_best.pth ...
2025-10-16 00:05:18,657 - train - INFO -     epoch          : 9
2025-10-16 00:05:18,660 - train - INFO -     loss           : 0.4456172960996628
2025-10-16 00:05:18,753 - train - INFO -     grad_norm      : 2.7257650566101073
2025-10-16 00:05:18,754 - train - INFO -     val_loss       : 0.7573878840934596
2025-10-16 00:05:18,755 - train - INFO -     val_CER_(Argmax): 0.17103310658530496
2025-10-16 00:05:18,756 - train - INFO -     val_WER_(Argmax): 0.48066185281193324
2025-10-16 00:05:18,757 - train - INFO -     test_loss      : 0.7295062383863984
2025-10-16 00:05:18,757 - train - INFO -     test_CER_(Argmax): 0.16261551521924106
2025-10-16 00:05:18,758 - train - INFO -     test_WER_(Argmax): 0.4628289488513657
2025-10-16 00:05:21,845 - train - INFO - Saving current best: model_best.pth ...
2025-10-16 00:22:59,057 - train - INFO -     epoch          : 10
2025-10-16 00:22:59,059 - train - INFO -     loss           : 0.41700817584991456
2025-10-16 00:22:59,060 - train - INFO -     grad_norm      : 2.59456018447876
2025-10-16 00:22:59,061 - train - INFO -     val_loss       : 0.7545174875202969
2025-10-16 00:22:59,064 - train - INFO -     val_CER_(Argmax): 0.16960430358929968
2025-10-16 00:22:59,065 - train - INFO -     val_WER_(Argmax): 0.48087200492757437
2025-10-16 00:22:59,067 - train - INFO -     test_loss      : 0.7289097205531306
2025-10-16 00:22:59,068 - train - INFO -     test_CER_(Argmax): 0.16165363621852386
2025-10-16 00:22:59,072 - train - INFO -     test_WER_(Argmax): 0.46350472768363055
2025-10-16 00:23:02,083 - train - INFO - Saving checkpoint: /home/jovyan/zenman67/training/proj/saved/beam_search_big_model/checkpoint-epoch10.pth ...