projecti7 commited on
Commit
7147656
·
verified ·
1 Parent(s): f13657f

Update latest checkpoint

Browse files
log/log-train-2026-01-13-11-24-40 ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-01-13 11:24:40,921 INFO [train.py:967] Training started
2
+ 2026-01-13 11:24:40,922 INFO [train.py:977] Device: cuda:0
3
+ 2026-01-13 11:24:40,925 INFO [train.py:986] {
4
+ "am_scale": 0.0,
5
+ "attention_dims": "192,192,192,192,192",
6
+ "average_period": 200,
7
+ "base_lr": 0.05,
8
+ "batch_idx_train": 0,
9
+ "best_train_epoch": -1,
10
+ "best_train_loss": Infinity,
11
+ "best_valid_epoch": -1,
12
+ "best_valid_loss": Infinity,
13
+ "blank_id": 0,
14
+ "bpe_model": "/kaggle/working/amharic_training/bpe/bpe.model",
15
+ "bucketing_sampler": true,
16
+ "cnn_module_kernels": "31,31,31,31,31",
17
+ "concatenate_cuts": false,
18
+ "context_size": 2,
19
+ "decode_chunk_len": 32,
20
+ "decoder_dim": 512,
21
+ "drop_last": true,
22
+ "duration_factor": 1.0,
23
+ "enable_musan": false,
24
+ "enable_spec_aug": true,
25
+ "encoder_dims": "384,384,384,384,384",
26
+ "encoder_unmasked_dims": "256,256,256,256,256",
27
+ "env_info": {
28
+ "IP address": "172.19.2.2",
29
+ "hostname": "8e64ffbd666a",
30
+ "icefall-git-branch": "master",
31
+ "icefall-git-date": "Fri Nov 28 03:42:20 2025",
32
+ "icefall-git-sha1": "0904e490-dirty",
33
+ "icefall-path": "/kaggle/working/icefall",
34
+ "k2-build-type": "Release",
35
+ "k2-git-date": "Thu Jul 25 03:34:26 2024",
36
+ "k2-git-sha1": "40e8d1676f6062e46458dc32ad21229c93cc9c50",
37
+ "k2-path": "/usr/local/lib/python3.12/dist-packages/k2/__init__.py",
38
+ "k2-version": "1.24.4",
39
+ "k2-with-cuda": true,
40
+ "lhotse-path": "/usr/local/lib/python3.12/dist-packages/lhotse/__init__.py",
41
+ "lhotse-version": "1.32.1",
42
+ "python-version": "3.12",
43
+ "torch-cuda-available": true,
44
+ "torch-cuda-version": "12.1",
45
+ "torch-version": "2.4.0+cu121"
46
+ },
47
+ "exp_dir": "/kaggle/working/amharic_training/exp_amharic_streaming",
48
+ "feature_dim": 80,
49
+ "feedforward_dims": "1024,1024,2048,2048,1024",
50
+ "full_libri": false,
51
+ "gap": 1.0,
52
+ "inf_check": false,
53
+ "input_strategy": "PrecomputedFeatures",
54
+ "joiner_dim": 512,
55
+ "keep_last_k": 1,
56
+ "lm_scale": 0.25,
57
+ "log_interval": 50,
58
+ "lr_batches": 5000,
59
+ "lr_epochs": 3.5,
60
+ "manifest_dir": "/kaggle/working/amharic_training/manifests",
61
+ "master_port": 12354,
62
+ "max_duration": 60,
63
+ "mini_libri": false,
64
+ "nhead": "8,8,8,8,8",
65
+ "num_buckets": 30,
66
+ "num_encoder_layers": "2,4,3,2,4",
67
+ "num_epochs": 50,
68
+ "num_left_chunks": 4,
69
+ "num_workers": 2,
70
+ "on_the_fly_feats": false,
71
+ "print_diagnostics": false,
72
+ "prune_range": 5,
73
+ "reset_interval": 200,
74
+ "return_cuts": true,
75
+ "save_every_n": 1000,
76
+ "seed": 42,
77
+ "short_chunk_size": 50,
78
+ "shuffle": true,
79
+ "simple_loss_scale": 0.5,
80
+ "spec_aug_time_warp_factor": 80,
81
+ "start_batch": 0,
82
+ "start_epoch": 1,
83
+ "subsampling_factor": 4,
84
+ "tensorboard": true,
85
+ "use_fp16": true,
86
+ "valid_interval": 1600,
87
+ "vocab_size": 1000,
88
+ "warm_step": 2000,
89
+ "world_size": 1,
90
+ "zipformer_downsampling_factors": "1,2,4,8,2"
91
+ }
92
+ 2026-01-13 11:24:40,925 INFO [train.py:988] About to create model
93
+ 2026-01-13 11:24:41,536 INFO [zipformer.py:405] At encoder stack 4, which has downsampling_factor=2, we will combine the outputs of layers 1 and 3, with downsampling_factors=2 and 8.
94
+ 2026-01-13 11:24:41,554 INFO [train.py:992] Number of model parameters: 71330891
95
+ 2026-01-13 11:24:43,824 INFO [asr_datamodule.py:422] About to get train-clean-100 cuts
96
+ 2026-01-13 11:24:43,825 INFO [asr_datamodule.py:239] Disable MUSAN
97
+ 2026-01-13 11:24:43,825 INFO [asr_datamodule.py:257] Enable SpecAugment
98
+ 2026-01-13 11:24:43,825 INFO [asr_datamodule.py:258] Time warp factor: 80
99
+ 2026-01-13 11:24:43,826 INFO [asr_datamodule.py:268] Num frame mask: 10
100
+ 2026-01-13 11:24:43,826 INFO [asr_datamodule.py:281] About to create train dataset
101
+ 2026-01-13 11:24:43,826 INFO [asr_datamodule.py:308] Using DynamicBucketingSampler.
102
+ 2026-01-13 11:24:44,140 INFO [asr_datamodule.py:324] About to create train dataloader
103
+ 2026-01-13 11:24:44,140 INFO [asr_datamodule.py:460] About to get dev-clean cuts
104
+ 2026-01-13 11:24:44,141 INFO [asr_datamodule.py:467] About to get dev-other cuts
105
+ 2026-01-13 11:24:44,141 INFO [asr_datamodule.py:355] About to create dev dataset
106
+ 2026-01-13 11:24:44,334 INFO [asr_datamodule.py:372] About to create dev dataloader
107
+ 2026-01-13 11:24:47,893 INFO [train.py:895] Epoch 1, batch 0, loss[loss=8.347, simple_loss=7.594, pruned_loss=7.506, over 1138.00 frames. ], tot_loss[loss=8.347, simple_loss=7.594, pruned_loss=7.506, over 1138.00 frames. ], batch size: 3, lr: 2.50e-02, grad_scale: 2.0
108
+ 2026-01-13 11:24:47,894 INFO [train.py:920] Computing validation loss
109
+ 2026-01-13 11:25:49,674 INFO [zipformer.py:2441] attn_weights_entropy = tensor([2.9199, 2.9204, 2.9209, 2.9167, 2.9196, 2.9205, 2.9204, 2.9204],
110
+ device='cuda:0'), covar=tensor([0.0048, 0.0085, 0.0084, 0.0041, 0.0051, 0.0054, 0.0087, 0.0047],
111
+ device='cuda:0'), in_proj_covar=tensor([0.0009, 0.0009, 0.0009, 0.0009, 0.0009, 0.0009, 0.0009, 0.0009],
112
+ device='cuda:0'), out_proj_covar=tensor([8.5571e-06, 8.6460e-06, 8.6548e-06, 8.5692e-06, 8.8457e-06, 8.6909e-06,
113
+ 8.7530e-06, 8.7241e-06], device='cuda:0')
114
+ 2026-01-13 11:26:30,538 INFO [zipformer.py:2441] attn_weights_entropy = tensor([4.2729, 4.2729, 4.2729, 4.2729, 4.2729, 4.2729, 4.2729, 4.2729],
115
+ device='cuda:0'), covar=tensor([0.0003, 0.0002, 0.0003, 0.0002, 0.0002, 0.0003, 0.0001, 0.0003],
116
+ device='cuda:0'), in_proj_covar=tensor([0.0009, 0.0009, 0.0009, 0.0009, 0.0009, 0.0009, 0.0009, 0.0008],
117
+ device='cuda:0'), out_proj_covar=tensor([8.8449e-06, 8.8559e-06, 8.7936e-06, 8.6492e-06, 8.7990e-06, 8.7099e-06,
118
+ 8.5965e-06, 8.7138e-06], device='cuda:0')
119
+ 2026-01-13 11:26:45,813 INFO [zipformer.py:2441] attn_weights_entropy = tensor([3.5795, 3.5857, 3.5854, 3.5868, 3.5840, 3.5854, 3.5849, 3.5870],
120
+ device='cuda:0'), covar=tensor([0.0071, 0.0040, 0.0091, 0.0069, 0.0067, 0.0080, 0.0082, 0.0090],
121
+ device='cuda:0'), in_proj_covar=tensor([0.0009, 0.0009, 0.0009, 0.0009, 0.0009, 0.0009, 0.0009, 0.0009],
122
+ device='cuda:0'), out_proj_covar=tensor([8.7006e-06, 8.7710e-06, 8.6193e-06, 8.7975e-06, 8.6463e-06, 8.7048e-06,
123
+ 8.7000e-06, 8.8221e-06], device='cuda:0')
tensorboard/events.out.tfevents.1768303480.8e64ffbd666a.84129.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f145c404960eccd40673635221acc99443610080542ee3d3d1f0f0bc8e34fcc9
3
+ size 598
tensorboard/events.out.tfevents.1768303480.8e64ffbd666a.84130.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b2c6a2446e890bbd3264a5e2bf1f87a91751443ad5639f3fab09e821cea2713
3
+ size 598