Add model.pt + model.py/config from AeiROBOT (funasr.AutoModel PyTorch support)

config.yaml

@@ -41,7 +41,7 @@ dataset_conf:
   batch_sampler: BatchSampler
   data_split_num: 1
   batch_type: token
-  batch_size: 6000
+  batch_size: 3200
   max_token_length: 2000
   min_token_length: 60
   max_source_length: 2000
@@ -49,7 +49,7 @@ dataset_conf:
   max_target_length: 200
   min_target_length: 0
   shuffle: true
-  num_workers: 4
+  num_workers: 8
   sos: 1
   eos: 2
   IndexDSJsonl: IndexDSJsonl
@@ -58,21 +58,21 @@ dataset_conf:
 train_conf:
   accum_grad: 1
   grad_clip: 5
-  max_epoch: 300
-  keep_nbest_models: 20
+  max_epoch: 1000
+  keep_nbest_models: 2
   avg_nbest_model: 10
   log_interval: 1
   resume: true
-  validate_interval: 2000
-  save_checkpoint_interval: 2000
+  validate_interval: 200
+  save_checkpoint_interval: 200
   use_deepspeed: false
-  deepspeed_config: /home/khw/Workspace/SenseVoice/deepspeed_conf/ds_stage1.json
+  deepspeed_config: /home/khw/workspace_tk/SenseVoice/deepspeed_conf/ds_stage1.json
 optim: adamw
 optim_conf:
   lr: 0.0002
 scheduler: warmuplr
 scheduler_conf:
-  warmup_steps: 25000
+  warmup_steps: 200
 specaug: SpecAugLFR
 specaug_conf:
   apply_time_warp: false
@@ -93,8 +93,8 @@ init_param: /home/khw/.cache/modelscope/hub/models/iic/SenseVoiceSmall/model.pt
 config: /home/khw/.cache/modelscope/hub/models/iic/SenseVoiceSmall/config.yaml
 is_training: true
 trust_remote_code: true
-train_data_set_list: /home/khw/Workspace/SenseVoice/data/train.jsonl
-valid_data_set_list: /home/khw/Workspace/SenseVoice/data/val.jsonl
+train_data_set_list: /home/khw/workspace_tk/SenseVoice/data/train_noise.jsonl
+valid_data_set_list: /home/khw/workspace_tk/SenseVoice/data/val.jsonl
 output_dir: ./outputs
 model_path: /home/khw/.cache/modelscope/hub/models/iic/SenseVoiceSmall
 device: cpu

model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:036bca0e9b512e41778ff6fc45d41ceee221b34cae4af36dee59bd234cb550e2
+size 2809160142

model.py

@@ -896,7 +896,7 @@ class SenseVoiceSmall(nn.Module):
                 align = ctc_forced_align(
                     logits_speech.unsqueeze(0).float(),
                     torch.Tensor(token_int[4:]).unsqueeze(0).long().to(logits_speech.device),
-                    (encoder_out_lens-4).long(),
+                    (encoder_out_lens-4).long()[i],
                     torch.tensor(len(token_int)-4).unsqueeze(0).long().to(logits_speech.device),
                     ignore_id=self.ignore_id,
                 )
@@ -907,7 +907,7 @@ class SenseVoiceSmall(nn.Module):
                 ts_max = encoder_out_lens[i] - 4
                 for pred_token, pred_frame in pred:
                     _end = _start + len(list(pred_frame))
-                    if pred_token != 0:
+                    if pred_token != 0 and token_id < len(tokens):
                         ts_left = max((_start*60-30)/1000, 0)
                         ts_right = min((_end*60-30)/1000, (ts_max*60-30)/1000)
                         timestamp.append([tokens[token_id], ts_left, ts_right])