File size: 11,579 Bytes
36b66b3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 | 2025-10-15 21:01:53,722 - train - INFO - ConformerModel(
(subsample): ConvSubsampling(
(conv): Sequential(
(0): Conv2d(1, 256, kernel_size=(3, 3), stride=(1, 2), padding=(1, 1))
(1): ReLU()
(2): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 2), padding=(1, 1))
(3): ReLU()
)
)
(pre_proj): LinearProjection(
(proj): Linear(in_features=65536, out_features=512, bias=True)
)
(pos_enc): RelativePositionalEncoding()
(blocks): ModuleList(
(0-15): 16 x ConformerBlock(
(ff1): FeedForwardModule(
(layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
(linear1): Linear(in_features=512, out_features=2048, bias=True)
(swish): Swish()
(dropout): Dropout(p=0.1, inplace=False)
(linear2): Linear(in_features=2048, out_features=512, bias=True)
(dropout2): Dropout(p=0.1, inplace=False)
)
(mhsa): MultiHeadSelfAttention(
(layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
(attn): MultiheadAttention(
(out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
)
(dropout): Dropout(p=0.1, inplace=False)
)
(conv): ConformerConvModule(
(layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
(pw_conv1): Conv1d(512, 1024, kernel_size=(1,), stride=(1,))
(glu): GLU(dim=1)
(dw_conv): Conv1d(512, 512, kernel_size=(31,), stride=(1,), padding=(15,), groups=512)
(bn): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(swish): Swish()
(pw_conv2): Conv1d(512, 512, kernel_size=(1,), stride=(1,))
(dropout): Dropout(p=0.1, inplace=False)
)
(ff2): FeedForwardModule(
(layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
(linear1): Linear(in_features=512, out_features=2048, bias=True)
(swish): Swish()
(dropout): Dropout(p=0.1, inplace=False)
(linear2): Linear(in_features=2048, out_features=512, bias=True)
(dropout2): Dropout(p=0.1, inplace=False)
)
(final_ln): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
)
)
(ctc_head): Linear(in_features=512, out_features=28, bias=True)
)
All parameters: 131720732
Trainable parameters: 131720732
2025-10-15 21:01:53,739 - pyctcdecode.alphabet - INFO - Alphabet determined to be of regular style.
2025-10-15 21:01:53,741 - pyctcdecode.alphabet - INFO - Alphabet determined to be of regular style.
2025-10-15 21:28:28,462 - train - INFO - epoch : 1
2025-10-15 21:28:28,464 - train - INFO - loss : 2.153041486740112
2025-10-15 21:28:28,465 - train - INFO - grad_norm : 1.188998259305954
2025-10-15 21:28:28,466 - train - INFO - val_loss : 2.172110017234757
2025-10-15 21:28:28,553 - train - INFO - val_CER_(Argmax): 0.5358357997251945
2025-10-15 21:28:28,554 - train - INFO - val_WER_(Argmax): 1.012158696859347
2025-10-15 21:28:28,555 - train - INFO - test_loss : 2.1243614394490313
2025-10-15 21:28:28,556 - train - INFO - test_CER_(Argmax): 0.5229153065123915
2025-10-15 21:28:28,557 - train - INFO - test_WER_(Argmax): 1.00157116613905
2025-10-15 21:28:31,531 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 21:50:34,559 - train - INFO - epoch : 2
2025-10-15 21:50:34,561 - train - INFO - loss : 1.4212745833396911
2025-10-15 21:50:34,562 - train - INFO - grad_norm : 1.9143898677825928
2025-10-15 21:50:34,563 - train - INFO - val_loss : 1.475996158532137
2025-10-15 21:50:34,564 - train - INFO - val_CER_(Argmax): 0.3734411473874421
2025-10-15 21:50:34,564 - train - INFO - val_WER_(Argmax): 0.8524215912740724
2025-10-15 21:50:34,565 - train - INFO - test_loss : 1.4299900364585039
2025-10-15 21:50:34,654 - train - INFO - test_CER_(Argmax): 0.36053000087408704
2025-10-15 21:50:34,655 - train - INFO - test_WER_(Argmax): 0.837277992722096
2025-10-15 21:50:37,890 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 22:11:52,360 - train - INFO - epoch : 3
2025-10-15 22:11:52,362 - train - INFO - loss : 1.1054761481285096
2025-10-15 22:11:52,363 - train - INFO - grad_norm : 2.133445930480957
2025-10-15 22:11:52,364 - train - INFO - val_loss : 1.179566823166503
2025-10-15 22:11:52,365 - train - INFO - val_CER_(Argmax): 0.3000637993717897
2025-10-15 22:11:52,366 - train - INFO - val_WER_(Argmax): 0.7330474730606038
2025-10-15 22:11:52,367 - train - INFO - test_loss : 1.1337390374846574
2025-10-15 22:11:52,368 - train - INFO - test_CER_(Argmax): 0.2865948095416102
2025-10-15 22:11:52,369 - train - INFO - test_WER_(Argmax): 0.7135280252571233
2025-10-15 22:11:55,916 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 22:32:39,457 - train - INFO - epoch : 4
2025-10-15 22:32:39,459 - train - INFO - loss : 0.8954937684535981
2025-10-15 22:32:39,460 - train - INFO - grad_norm : 2.265110855102539
2025-10-15 22:32:39,462 - train - INFO - val_loss : 1.0523416495182105
2025-10-15 22:32:39,462 - train - INFO - val_CER_(Argmax): 0.26451346666147674
2025-10-15 22:32:39,464 - train - INFO - val_WER_(Argmax): 0.6780852042352846
2025-10-15 22:32:39,465 - train - INFO - test_loss : 1.0155282700207175
2025-10-15 22:32:39,465 - train - INFO - test_CER_(Argmax): 0.25441022118607404
2025-10-15 22:32:39,466 - train - INFO - test_WER_(Argmax): 0.6582085265629509
2025-10-15 22:32:42,588 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 22:52:16,859 - train - INFO - epoch : 5
2025-10-15 22:52:16,861 - train - INFO - loss : 0.7874286377429962
2025-10-15 22:52:16,862 - train - INFO - grad_norm : 2.3151290917396548
2025-10-15 22:52:16,863 - train - INFO - val_loss : 0.921425876532786
2025-10-15 22:52:16,864 - train - INFO - val_CER_(Argmax): 0.22771190898381688
2025-10-15 22:52:16,864 - train - INFO - val_WER_(Argmax): 0.6050087899838589
2025-10-15 22:52:16,865 - train - INFO - test_loss : 0.8859942057510701
2025-10-15 22:52:16,866 - train - INFO - test_CER_(Argmax): 0.21756451149393294
2025-10-15 22:52:16,867 - train - INFO - test_WER_(Argmax): 0.5850164614033462
2025-10-15 22:52:19,966 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 22:59:22,179 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ReadTimeoutError("HTTPSConnectionPool(host='www.comet.com', port=443): Read timed out. (read timeout=10)")': /clientlib/batch/logger/experiment/metric
2025-10-15 23:00:32,468 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ReadTimeoutError("HTTPSConnectionPool(host='www.comet.com', port=443): Read timed out. (read timeout=10)")': /clientlib/rest/v2/write/experiment/output
2025-10-15 23:10:58,659 - train - INFO - epoch : 6
2025-10-15 23:10:58,663 - train - INFO - loss : 0.603953384757042
2025-10-15 23:10:58,668 - train - INFO - grad_norm : 2.3825424408912657
2025-10-15 23:10:58,673 - train - INFO - val_loss : 0.8432797542690526
2025-10-15 23:10:58,674 - train - INFO - val_CER_(Argmax): 0.19975002812164636
2025-10-15 23:10:58,678 - train - INFO - val_WER_(Argmax): 0.5460640429402533
2025-10-15 23:10:58,679 - train - INFO - test_loss : 0.8142017166062099
2025-10-15 23:10:58,683 - train - INFO - test_CER_(Argmax): 0.1905588444625169
2025-10-15 23:10:58,687 - train - INFO - test_WER_(Argmax): 0.5286493462729215
2025-10-15 23:11:01,982 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 23:15:33,860 - urllib3.connectionpool - WARNING - Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'ReadTimeoutError("HTTPSConnectionPool(host='www.comet.com', port=443): Read timed out. (read timeout=10)")': /clientlib/batch/logger/experiment/metric
2025-10-15 23:29:41,360 - train - INFO - epoch : 7
2025-10-15 23:29:41,362 - train - INFO - loss : 0.5495029705762863
2025-10-15 23:29:41,363 - train - INFO - grad_norm : 2.781264133453369
2025-10-15 23:29:41,364 - train - INFO - val_loss : 0.7903630255594761
2025-10-15 23:29:41,365 - train - INFO - val_CER_(Argmax): 0.18955925445322028
2025-10-15 23:29:41,366 - train - INFO - val_WER_(Argmax): 0.5232188275226236
2025-10-15 23:29:41,367 - train - INFO - test_loss : 0.7601223377192893
2025-10-15 23:29:41,368 - train - INFO - test_CER_(Argmax): 0.1790136937010882
2025-10-15 23:29:41,368 - train - INFO - test_WER_(Argmax): 0.5031266710478192
2025-10-15 23:29:44,559 - train - INFO - Saving current best: model_best.pth ...
2025-10-15 23:47:30,958 - train - INFO - epoch : 8
2025-10-15 23:47:30,960 - train - INFO - loss : 0.4621651893854141
2025-10-15 23:47:30,961 - train - INFO - grad_norm : 2.654181697368622
2025-10-15 23:47:30,962 - train - INFO - val_loss : 0.7660906538808134
2025-10-15 23:47:30,962 - train - INFO - val_CER_(Argmax): 0.17487329416638675
2025-10-15 23:47:30,963 - train - INFO - val_WER_(Argmax): 0.4908353581541641
2025-10-15 23:47:30,964 - train - INFO - test_loss : 0.7469499325606881
2025-10-15 23:47:30,965 - train - INFO - test_CER_(Argmax): 0.16683605401739357
2025-10-15 23:47:30,965 - train - INFO - test_WER_(Argmax): 0.4732129735921914
2025-10-15 23:47:34,214 - train - INFO - Saving current best: model_best.pth ...
2025-10-16 00:05:18,657 - train - INFO - epoch : 9
2025-10-16 00:05:18,660 - train - INFO - loss : 0.4456172960996628
2025-10-16 00:05:18,753 - train - INFO - grad_norm : 2.7257650566101073
2025-10-16 00:05:18,754 - train - INFO - val_loss : 0.7573878840934596
2025-10-16 00:05:18,755 - train - INFO - val_CER_(Argmax): 0.17103310658530496
2025-10-16 00:05:18,756 - train - INFO - val_WER_(Argmax): 0.48066185281193324
2025-10-16 00:05:18,757 - train - INFO - test_loss : 0.7295062383863984
2025-10-16 00:05:18,757 - train - INFO - test_CER_(Argmax): 0.16261551521924106
2025-10-16 00:05:18,758 - train - INFO - test_WER_(Argmax): 0.4628289488513657
2025-10-16 00:05:21,845 - train - INFO - Saving current best: model_best.pth ...
2025-10-16 00:22:59,057 - train - INFO - epoch : 10
2025-10-16 00:22:59,059 - train - INFO - loss : 0.41700817584991456
2025-10-16 00:22:59,060 - train - INFO - grad_norm : 2.59456018447876
2025-10-16 00:22:59,061 - train - INFO - val_loss : 0.7545174875202969
2025-10-16 00:22:59,064 - train - INFO - val_CER_(Argmax): 0.16960430358929968
2025-10-16 00:22:59,065 - train - INFO - val_WER_(Argmax): 0.48087200492757437
2025-10-16 00:22:59,067 - train - INFO - test_loss : 0.7289097205531306
2025-10-16 00:22:59,068 - train - INFO - test_CER_(Argmax): 0.16165363621852386
2025-10-16 00:22:59,072 - train - INFO - test_WER_(Argmax): 0.46350472768363055
2025-10-16 00:23:02,083 - train - INFO - Saving checkpoint: /home/jovyan/zenman67/training/proj/saved/beam_search_big_model/checkpoint-epoch10.pth ...
|