| | --- |
| | tags: |
| | - espnet |
| | - audio |
| | - automatic-speech-recognition |
| | language: en |
| | datasets: |
| | - slurp_entity |
| | license: cc-by-4.0 |
| | --- |
| | |
| | ## ESPnet2 ASR model |
| |
|
| | ### `pyf98/slurp_entity_e_branchformer` |
| | |
| | This model was trained by Yifan Peng using slurp_entity recipe in [espnet](https://github.com/espnet/espnet/). |
| |
|
| | References: |
| | - [E-Branchformer: Branchformer with Enhanced merging for speech recognition (SLT 2022)](https://arxiv.org/abs/2210.00077) |
| | - [Branchformer: Parallel MLP-Attention Architectures to Capture Local and Global Context for Speech Recognition and Understanding (ICML 2022)](https://proceedings.mlr.press/v162/peng22a.html) |
| |
|
| | ### Demo: How to use in ESPnet2 |
| |
|
| | Follow the [ESPnet installation instructions](https://espnet.github.io/espnet/installation.html) |
| | if you haven't done that already. |
| |
|
| | ```bash |
| | cd espnet |
| | git checkout 4bbd29a40cc7e2259996d30c0c76d3d789c1153d |
| | pip install -e . |
| | cd egs2/slurp_entity/asr1 |
| | ./run.sh --skip_data_prep false --skip_train true --download_model pyf98/slurp_entity_e_branchformer |
| | ``` |
| |
|
| | <!-- Generated by scripts/utils/show_asr_result.sh --> |
| | # RESULTS |
| | ## Environments |
| | - date: `Mon Feb 27 19:14:30 CST 2023` |
| | - python version: `3.9.15 (main, Nov 24 2022, 14:31:59) [GCC 11.2.0]` |
| | - espnet version: `espnet 202301` |
| | - pytorch version: `pytorch 1.13.1` |
| | - Git hash: `4bbd29a40cc7e2259996d30c0c76d3d789c1153d` |
| | - Commit date: `Sat Feb 25 21:54:03 2023 -0600` |
| |
|
| | ## exp/asr_train_asr_e_branchformer_e12_mlp3072_linear1024_layerdrop_raw_en_word |
| | ### WER |
| | |
| | |dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| |
| | |---|---|---|---|---|---|---|---|---| |
| | |decode_asr_asr_model_valid.acc.ave_10best/devel|8690|178058|84.6|7.6|7.8|3.2|18.6|51.2| |
| | |decode_asr_asr_model_valid.acc.ave_10best/test|13078|262176|83.7|7.7|8.6|3.0|19.3|49.7| |
| | |
| | ### CER |
| | |
| | |dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| |
| | |---|---|---|---|---|---|---|---|---| |
| | |decode_asr_asr_model_valid.acc.ave_10best/devel|8690|847400|90.8|3.0|6.2|3.5|12.7|51.2| |
| | |decode_asr_asr_model_valid.acc.ave_10best/test|13078|1245475|89.7|3.1|7.2|3.4|13.6|49.7| |
| | |
| | |
| | ### Intent Classification |
| | |
| | - Valid Intent Classification Result: |
| | 0.8781357882623706 |
| | - Test Intent Classification Result: |
| | 0.8743691695977979 |
| | |
| | ### Entity |
| | |
| | |Slu f1|Precision|Recall|F-Measure| |
| | |:---:|:---:|:---:|:---:| |
| | | test | 0.7940 | 0.7582 | 0.7757 | |
| | |
| | |
| | |
| | ## ASR config |
| | |
| | <details><summary>expand</summary> |
| | |
| | ``` |
| | config: conf/tuning/train_asr_e_branchformer_e12_mlp3072_linear1024_layerdrop.yaml |
| | print_config: false |
| | log_level: INFO |
| | dry_run: false |
| | iterator_type: sequence |
| | output_dir: exp/asr_train_asr_e_branchformer_e12_mlp3072_linear1024_layerdrop_raw_en_word |
| | ngpu: 1 |
| | seed: 0 |
| | num_workers: 1 |
| | num_att_plot: 3 |
| | dist_backend: nccl |
| | dist_init_method: env:// |
| | dist_world_size: null |
| | dist_rank: null |
| | local_rank: 0 |
| | dist_master_addr: null |
| | dist_master_port: null |
| | dist_launcher: null |
| | multiprocessing_distributed: false |
| | unused_parameters: false |
| | sharded_ddp: false |
| | cudnn_enabled: true |
| | cudnn_benchmark: false |
| | cudnn_deterministic: true |
| | collect_stats: false |
| | write_collected_feats: false |
| | max_epoch: 60 |
| | patience: null |
| | val_scheduler_criterion: |
| | - valid |
| | - loss |
| | early_stopping_criterion: |
| | - valid |
| | - loss |
| | - min |
| | best_model_criterion: |
| | - - valid |
| | - acc |
| | - max |
| | keep_nbest_models: 10 |
| | nbest_averaging_interval: 0 |
| | grad_clip: 5.0 |
| | grad_clip_type: 2.0 |
| | grad_noise: false |
| | accum_grad: 1 |
| | no_forward_run: false |
| | resume: true |
| | train_dtype: float32 |
| | use_amp: false |
| | log_interval: null |
| | use_matplotlib: true |
| | use_tensorboard: true |
| | create_graph_in_tensorboard: false |
| | use_wandb: false |
| | wandb_project: null |
| | wandb_id: null |
| | wandb_entity: null |
| | wandb_name: null |
| | wandb_model_log_interval: -1 |
| | detect_anomaly: false |
| | pretrain_path: null |
| | init_param: [] |
| | ignore_init_mismatch: false |
| | freeze_param: [] |
| | num_iters_per_epoch: null |
| | batch_size: 64 |
| | valid_batch_size: null |
| | batch_bins: 1000000 |
| | valid_batch_bins: null |
| | train_shape_file: |
| | - exp/asr_stats_raw_en_word/train/speech_shape |
| | - exp/asr_stats_raw_en_word/train/text_shape.word |
| | valid_shape_file: |
| | - exp/asr_stats_raw_en_word/valid/speech_shape |
| | - exp/asr_stats_raw_en_word/valid/text_shape.word |
| | batch_type: folded |
| | valid_batch_type: null |
| | fold_length: |
| | - 80000 |
| | - 150 |
| | sort_in_batch: descending |
| | sort_batch: descending |
| | multiple_iterator: false |
| | chunk_length: 500 |
| | chunk_shift_ratio: 0.5 |
| | num_cache_chunks: 1024 |
| | train_data_path_and_name_and_type: |
| | - - dump/raw/train/wav.scp |
| | - speech |
| | - kaldi_ark |
| | - - dump/raw/train/text |
| | - text |
| | - text |
| | valid_data_path_and_name_and_type: |
| | - - dump/raw/devel/wav.scp |
| | - speech |
| | - kaldi_ark |
| | - - dump/raw/devel/text |
| | - text |
| | - text |
| | allow_variable_data_keys: false |
| | max_cache_size: 0.0 |
| | max_cache_fd: 32 |
| | valid_max_cache_size: null |
| | exclude_weight_decay: false |
| | exclude_weight_decay_conf: {} |
| | optim: adam |
| | optim_conf: |
| | lr: 0.001 |
| | weight_decay: 1.0e-06 |
| | scheduler: warmuplr |
| | scheduler_conf: |
| | warmup_steps: 35000 |
| | token_list: |
| | - <blank> |
| | - <unk> |
| | - ▁SEP |
| | - ▁FILL |
| | - s |
| | - ▁the |
| | - a |
| | - ▁to |
| | - ▁i |
| | - ▁me |
| | - e |
| | - ▁s |
| | - ▁a |
| | - i |
| | - ▁you |
| | - ▁what |
| | - er |
| | - ing |
| | - u |
| | - ▁is |
| | - '''' |
| | - o |
| | - p |
| | - ▁in |
| | - ▁p |
| | - y |
| | - ▁my |
| | - ▁please |
| | - d |
| | - c |
| | - m |
| | - ▁b |
| | - l |
| | - ▁m |
| | - ▁c |
| | - st |
| | - date |
| | - n |
| | - ▁d |
| | - le |
| | - b |
| | - ▁for |
| | - re |
| | - t |
| | - ▁on |
| | - en |
| | - h |
| | - 'on' |
| | - ar |
| | - person |
| | - ▁re |
| | - ▁f |
| | - ▁g |
| | - ▁of |
| | - an |
| | - ▁ |
| | - g |
| | - ▁today |
| | - ▁t |
| | - or |
| | - ▁it |
| | - ▁this |
| | - ▁h |
| | - r |
| | - f |
| | - at |
| | - ch |
| | - ce |
| | - place_name |
| | - ▁email |
| | - ▁do |
| | - es |
| | - ri |
| | - ▁e |
| | - ▁w |
| | - ic |
| | - in |
| | - ▁that |
| | - event_name |
| | - ▁play |
| | - ▁and |
| | - al |
| | - ▁n |
| | - ▁can |
| | - email_query |
| | - ve |
| | - ▁new |
| | - day |
| | - it |
| | - ate |
| | - ▁from |
| | - ▁have |
| | - k |
| | - time |
| | - ▁am |
| | - media_type |
| | - email_sendemail |
| | - ent |
| | - ▁olly |
| | - qa_factoid |
| | - se |
| | - v |
| | - et |
| | - ck |
| | - ▁any |
| | - calendar_set |
| | - ly |
| | - th |
| | - ▁how |
| | - ▁meeting |
| | - ed |
| | - ▁tell |
| | - ▁st |
| | - x |
| | - ur |
| | - ro |
| | - ▁at |
| | - nd |
| | - ▁list |
| | - w |
| | - ▁u |
| | - ou |
| | - ▁not |
| | - ▁about |
| | - ▁an |
| | - ▁o |
| | - general_negate |
| | - ut |
| | - ▁time |
| | - ▁be |
| | - ▁ch |
| | - ▁are |
| | - social_post |
| | - business_name |
| | - la |
| | - ty |
| | - play_music |
| | - ot |
| | - general_quirky |
| | - ▁l |
| | - ▁sh |
| | - ▁tweet |
| | - om |
| | - ▁week |
| | - um |
| | - ▁one |
| | - ter |
| | - ▁he |
| | - ▁up |
| | - ▁com |
| | - general_praise |
| | - weather_query |
| | - ▁next |
| | - ▁th |
| | - ▁check |
| | - calendar_query |
| | - ▁last |
| | - ▁ro |
| | - ad |
| | - is |
| | - ▁with |
| | - ay |
| | - ▁send |
| | - pe |
| | - ▁pm |
| | - ▁tomorrow |
| | - ▁j |
| | - un |
| | - ▁train |
| | - general_explain |
| | - ▁v |
| | - one |
| | - ▁r |
| | - ra |
| | - news_query |
| | - ation |
| | - ▁emails |
| | - us |
| | - if |
| | - ct |
| | - ▁co |
| | - ▁add |
| | - ▁will |
| | - ▁se |
| | - nt |
| | - ▁was |
| | - ine |
| | - ▁de |
| | - ▁set |
| | - ▁ex |
| | - ▁would |
| | - ir |
| | - ow |
| | - ber |
| | - general_repeat |
| | - ight |
| | - ook |
| | - ▁again |
| | - ▁song |
| | - currency_name |
| | - ll |
| | - ▁ha |
| | - ▁go |
| | - relation |
| | - te |
| | - ion |
| | - and |
| | - ▁y |
| | - ▁ye |
| | - general_affirm |
| | - general_confirm |
| | - ery |
| | - ▁po |
| | - ff |
| | - ▁we |
| | - ▁turn |
| | - ▁did |
| | - ▁mar |
| | - ▁alarm |
| | - ▁like |
| | - datetime_query |
| | - ers |
| | - ▁all |
| | - ▁remind |
| | - ▁so |
| | - qa_definition |
| | - ▁calendar |
| | - end |
| | - ▁said |
| | - ci |
| | - ▁off |
| | - ▁john |
| | - ▁day |
| | - ss |
| | - pla |
| | - ume |
| | - ▁get |
| | - ail |
| | - pp |
| | - z |
| | - ry |
| | - am |
| | - ▁need |
| | - as |
| | - ▁thank |
| | - ▁wh |
| | - ▁want |
| | - ▁right |
| | - ▁jo |
| | - ▁facebook |
| | - ▁k |
| | - ge |
| | - ld |
| | - ▁fri |
| | - ▁two |
| | - general_dontcare |
| | - ▁news |
| | - ol |
| | - oo |
| | - ant |
| | - ▁five |
| | - ▁event |
| | - ake |
| | - definition_word |
| | - transport_type |
| | - ▁your |
| | - vi |
| | - orn |
| | - op |
| | - ▁weather |
| | - ome |
| | - ▁app |
| | - ▁lo |
| | - de |
| | - ▁music |
| | - weather_descriptor |
| | - ak |
| | - ke |
| | - ▁there |
| | - ▁si |
| | - ▁lights |
| | - ▁now |
| | - ▁mo |
| | - calendar_remove |
| | - our |
| | - ▁dollar |
| | - food_type |
| | - me |
| | - ▁more |
| | - ▁no |
| | - ▁birthday |
| | - orrect |
| | - ▁rep |
| | - ▁show |
| | - play_radio |
| | - ▁mon |
| | - ▁does |
| | - ood |
| | - ag |
| | - li |
| | - ▁sto |
| | - ▁contact |
| | - cket |
| | - email_querycontact |
| | - ▁ev |
| | - ▁could |
| | - ange |
| | - ▁just |
| | - out |
| | - ame |
| | - . |
| | - ▁ja |
| | - ▁confirm |
| | - qa_currency |
| | - ▁man |
| | - ▁late |
| | - ▁think |
| | - ▁some |
| | - timeofday |
| | - ▁bo |
| | - qa_stock |
| | - ong |
| | - ▁start |
| | - ▁work |
| | - ▁ten |
| | - int |
| | - ▁command |
| | - all |
| | - ▁make |
| | - ▁la |
| | - j |
| | - ▁answ |
| | - ▁hour |
| | - ▁cle |
| | - ah |
| | - ▁find |
| | - ▁service |
| | - ▁fa |
| | - qu |
| | - general_commandstop |
| | - ai |
| | - ▁when |
| | - ▁te |
| | - ▁by |
| | - social_query |
| | - ard |
| | - ▁tw |
| | - ul |
| | - id |
| | - ▁seven |
| | - ▁where |
| | - ▁much |
| | - art |
| | - ▁appointment |
| | - ver |
| | - artist_name |
| | - el |
| | - device_type |
| | - ▁know |
| | - ▁three |
| | - ▁events |
| | - ▁tr |
| | - ▁li |
| | - ork |
| | - red |
| | - ect |
| | - ▁let |
| | - ▁respon |
| | - ▁par |
| | - zz |
| | - ▁give |
| | - ▁twenty |
| | - ▁ti |
| | - ▁curre |
| | - play_podcasts |
| | - ▁radio |
| | - cooking_recipe |
| | - transport_query |
| | - ▁con |
| | - gh |
| | - ▁le |
| | - lists_query |
| | - ▁rem |
| | - recommendation_events |
| | - house_place |
| | - alarm_set |
| | - play_audiobook |
| | - ist |
| | - ase |
| | - music_genre |
| | - ive |
| | - ast |
| | - player_setting |
| | - ort |
| | - lly |
| | - news_topic |
| | - list_name |
| | - ▁playlist |
| | - ▁ne |
| | - business_type |
| | - personal_info |
| | - ind |
| | - ust |
| | - di |
| | - ress |
| | - recommendation_locations |
| | - lists_createoradd |
| | - iot_hue_lightoff |
| | - lists_remove |
| | - ord |
| | - ▁light |
| | - ere |
| | - alarm_query |
| | - audio_volume_mute |
| | - music_query |
| | - ▁audio |
| | - rain |
| | - ▁date |
| | - ▁order |
| | - audio_volume_up |
| | - ▁ar |
| | - ▁podcast |
| | - transport_ticket |
| | - mail |
| | - iot_hue_lightchange |
| | - iot_coffee |
| | - radio_name |
| | - ill |
| | - ▁ri |
| | - '@' |
| | - takeaway_query |
| | - song_name |
| | - takeaway_order |
| | - ▁ra |
| | - email_addcontact |
| | - play_game |
| | - book |
| | - transport_traffic |
| | - ▁house |
| | - music_likeness |
| | - her |
| | - transport_taxi |
| | - iot_hue_lightdim |
| | - ment |
| | - ght |
| | - fo |
| | - order_type |
| | - color_type |
| | - '1' |
| | - ven |
| | - ould |
| | - general_joke |
| | - ess |
| | - ain |
| | - qa_maths |
| | - ▁place |
| | - ▁twe |
| | - cast |
| | - iot_cleaning |
| | - ▁che |
| | - ▁cont |
| | - ith |
| | - audiobook_name |
| | - email_address |
| | - game_name |
| | - ▁cal |
| | - general_frequency |
| | - ▁tom |
| | - ▁food |
| | - act |
| | - iot_hue_lightup |
| | - '2' |
| | - alarm_remove |
| | - podcast_descriptor |
| | - ▁definition |
| | - audio_volume_down |
| | - ▁media |
| | - email_folder |
| | - dia |
| | - meal_type |
| | - ▁mus |
| | - recommendation_movies |
| | - ▁ad |
| | - ree |
| | - pt |
| | - now |
| | - playlist_name |
| | - ▁person |
| | - change_amount |
| | - ▁pla |
| | - escri |
| | - datetime_convert |
| | - podcast_name |
| | - ▁ab |
| | - time_zone |
| | - ▁def |
| | - ting |
| | - iot_wemo_on |
| | - music_settings |
| | - iot_wemo_off |
| | - orre |
| | - cy |
| | - ank |
| | - music_descriptor |
| | - lar |
| | - app_name |
| | - row |
| | - joke_type |
| | - xt |
| | - of |
| | - ition |
| | - ▁meet |
| | - ink |
| | - ▁confir |
| | - transport_agency |
| | - general_greet |
| | - ▁business |
| | - ▁art |
| | - ▁ag |
| | - urn |
| | - escript |
| | - rom |
| | - ▁rel |
| | - ▁au |
| | - ▁currency |
| | - audio_volume_other |
| | - iot_hue_lighton |
| | - ▁artist |
| | - '?' |
| | - ▁bus |
| | - cooking_type |
| | - movie_name |
| | - coffee_type |
| | - ingredient |
| | - ather |
| | - music_dislikeness |
| | - sp |
| | - q |
| | - ▁ser |
| | - esc |
| | - ▁bir |
| | - ▁cur |
| | - name |
| | - ▁tran |
| | - ▁hou |
| | - ek |
| | - uch |
| | - ▁conf |
| | - ▁face |
| | - '9' |
| | - ▁birth |
| | - I |
| | - sw |
| | - transport_descriptor |
| | - ▁comm |
| | - lease |
| | - transport_name |
| | - aid |
| | - movie_type |
| | - ▁device |
| | - alarm_type |
| | - audiobook_author |
| | - '5' |
| | - drink_type |
| | - ▁joh |
| | - ▁defin |
| | - word |
| | - ▁curren |
| | - order |
| | - iness |
| | - W |
| | - cooking_query |
| | - sport_type |
| | - ▁relation |
| | - oint |
| | - H |
| | - '8' |
| | - A |
| | - '0' |
| | - ▁dol |
| | - vice |
| | - ▁pers |
| | - '&' |
| | - T |
| | - ▁appoint |
| | - _ |
| | - '7' |
| | - '3' |
| | - '-' |
| | - game_type |
| | - ▁pod |
| | - N |
| | - M |
| | - E |
| | - list |
| | - music_album |
| | - dio |
| | - ▁transport |
| | - qa_query |
| | - C |
| | - O |
| | - U |
| | - query_detail |
| | - ']' |
| | - '[' |
| | - descriptor |
| | - ':' |
| | - spon |
| | - <sos/eos> |
| | init: null |
| | input_size: null |
| | ctc_conf: |
| | dropout_rate: 0.0 |
| | ctc_type: builtin |
| | reduce: true |
| | ignore_nan_grad: null |
| | zero_infinity: true |
| | joint_net_conf: null |
| | use_preprocessor: true |
| | token_type: word |
| | bpemodel: null |
| | non_linguistic_symbols: null |
| | cleaner: null |
| | g2p: null |
| | speech_volume_normalize: null |
| | rir_scp: null |
| | rir_apply_prob: 1.0 |
| | noise_scp: null |
| | noise_apply_prob: 1.0 |
| | noise_db_range: '13_15' |
| | short_noise_thres: 0.5 |
| | aux_ctc_tasks: [] |
| | frontend: default |
| | frontend_conf: |
| | fs: 16k |
| | specaug: specaug |
| | specaug_conf: |
| | apply_time_warp: true |
| | time_warp_window: 5 |
| | time_warp_mode: bicubic |
| | apply_freq_mask: true |
| | freq_mask_width_range: |
| | - 0 |
| | - 30 |
| | num_freq_mask: 2 |
| | apply_time_mask: true |
| | time_mask_width_range: |
| | - 0 |
| | - 40 |
| | num_time_mask: 2 |
| | normalize: utterance_mvn |
| | normalize_conf: {} |
| | model: espnet |
| | model_conf: |
| | ctc_weight: 0.3 |
| | lsm_weight: 0.1 |
| | length_normalized_loss: false |
| | extract_feats_in_collect_stats: false |
| | preencoder: null |
| | preencoder_conf: {} |
| | encoder: e_branchformer |
| | encoder_conf: |
| | output_size: 512 |
| | attention_heads: 8 |
| | attention_layer_type: rel_selfattn |
| | pos_enc_layer_type: rel_pos |
| | rel_pos_type: latest |
| | cgmlp_linear_units: 3072 |
| | cgmlp_conv_kernel: 31 |
| | use_linear_after_conv: false |
| | gate_activation: identity |
| | num_blocks: 12 |
| | dropout_rate: 0.1 |
| | positional_dropout_rate: 0.1 |
| | attention_dropout_rate: 0.1 |
| | input_layer: conv2d |
| | layer_drop_rate: 0.1 |
| | linear_units: 1024 |
| | positionwise_layer_type: linear |
| | macaron_ffn: true |
| | use_ffn: true |
| | merge_conv_kernel: 31 |
| | postencoder: null |
| | postencoder_conf: {} |
| | decoder: transformer |
| | decoder_conf: |
| | attention_heads: 8 |
| | linear_units: 2048 |
| | num_blocks: 6 |
| | dropout_rate: 0.1 |
| | positional_dropout_rate: 0.1 |
| | self_attention_dropout_rate: 0.1 |
| | src_attention_dropout_rate: 0.1 |
| | layer_drop_rate: 0.2 |
| | preprocessor: default |
| | preprocessor_conf: {} |
| | required: |
| | - output_dir |
| | - token_list |
| | version: '202301' |
| | distributed: false |
| | ``` |
| | |
| | </details> |
| |
|
| |
|
| |
|
| | ### Citing ESPnet |
| |
|
| | ```BibTex |
| | @inproceedings{watanabe2018espnet, |
| | author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai}, |
| | title={{ESPnet}: End-to-End Speech Processing Toolkit}, |
| | year={2018}, |
| | booktitle={Proceedings of Interspeech}, |
| | pages={2207--2211}, |
| | doi={10.21437/Interspeech.2018-1456}, |
| | url={http://dx.doi.org/10.21437/Interspeech.2018-1456} |
| | } |
| | |
| | |
| | |
| | |
| | ``` |
| |
|
| | or arXiv: |
| |
|
| | ```bibtex |
| | @misc{watanabe2018espnet, |
| | title={ESPnet: End-to-End Speech Processing Toolkit}, |
| | author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai}, |
| | year={2018}, |
| | eprint={1804.00015}, |
| | archivePrefix={arXiv}, |
| | primaryClass={cs.CL} |
| | } |
| | ``` |
| |
|
