Add Task Specifier model

add_tokens-Copy1.txt

@@ -0,0 +1,460 @@
+command:yes
+command:down
+command:no
+command:stop
+command:go
+command:on
+command:left
+command:right
+command:_unknown_
+command:_silence_
+command:off
+command:up
+em:neu
+em:ang
+em:sad
+em:hap
+em:oth
+in:increase_heat_washroom
+in:deactivate_lights_none
+in:deactivate_lights_bedroom
+in:decrease_heat_none
+in:deactivate_lights_kitchen
+in:change_language_none_none
+in:activate_music_none
+in:change_language_English_none
+in:activate_lights_none
+in:deactivate_lights_washroom
+in:change_language_German_none
+in:decrease_heat_kitchen
+in:increase_volume_none
+in:decrease_heat_bedroom
+in:deactivate_music_none
+in:decrease_volume_none
+in:change_language_Chinese_none
+in:decrease_heat_washroom
+in:change_language_Korean_none
+in:increase_heat_none
+in:bring_newspaper_none
+in:activate_lamp_none
+in:deactivate_lamp_none
+in:bring_juice_none
+in:activate_lights_kitchen
+in:increase_heat_kitchen
+in:bring_socks_none
+in:activate_lights_bedroom
+in:increase_heat_bedroom
+in:activate_lights_washroom
+in:bring_shoes_none
+command:A
+command:B
+command:C
+command:D
+command:E
+command:F
+command:0
+command:1
+command:2
+command:3
+command:4
+command:5
+command:6
+command:7
+command:8
+command:9
+accent:american
+accent:australian
+accent:bangla
+accent:british
+accent:indian
+accent:malayalam
+accent:odiya
+accent:telugu
+accent:welsh
+class:spoof
+class:bonafide
+lang:ru
+lang:es
+lang:it
+lang:en
+lang:fr
+lang:de
+class:sarcasm
+class:not_sarcasm
+gender:f
+gender:m
+audio_class:43
+audio_class:1
+audio_class:10
+audio_class:17
+audio_class:37
+audio_class:35
+audio_class:29
+audio_class:11
+audio_class:3
+audio_class:19
+audio_class:32
+audio_class:16
+audio_class:47
+audio_class:44
+audio_class:13
+audio_class:36
+audio_class:39
+audio_class:20
+audio_class:24
+audio_class:14
+audio_class:9
+audio_class:26
+audio_class:5
+audio_class:28
+audio_class:8
+audio_class:30
+audio_class:0
+audio_class:48
+audio_class:21
+audio_class:31
+audio_class:38
+audio_class:6
+audio_class:45
+audio_class:33
+audio_class:15
+audio_class:34
+audio_class:2
+audio_class:7
+audio_class:49
+audio_class:12
+audio_class:40
+audio_class:25
+audio_class:22
+audio_class:4
+audio_class:42
+audio_class:41
+audio_class:23
+audio_class:46
+audio_class:18
+audio_class:27
+vad_class:speech
+<|audio|>
+<|ic|>
+<|scr|>
+<|er|>
+<|accent_rec|>
+<|fsd|>
+<|lid|>
+<|scd|>
+<|gid|>
+<|auc|>
+<|vad|>
+<|google_scr|>
+<|grabo_scr|>
+<|lt_scr|>
+<|ar_scr|>
+<|fsc|>
+<|voxforge|>
+<|asvspoof|>
+<|iemocap|>
+<|accentdb|>
+<|mustard|>
+<|mustard_plus_plus|>
+<|voxceleb|>
+<|esc50|>
+<|freesound|>
+in:increasebrightness
+in:setlightcolor
+in:setlightbrightness
+in:switchlighton
+in:decreasebrightness
+in:switchlightoff
+in:music_likeness
+in:music_settings
+in:play_music
+in:iot_hue_lightoff
+in:play_radio
+in:weather_query
+in:cooking_recipe
+in:email_sendemail
+in:calendar_set
+in:lists_remove
+in:play_podcasts
+in:qa_definition
+in:audio_volume_up
+in:news_query
+in:general_quirky
+in:email_query
+in:audio_volume_down
+in:takeaway_query
+in:general_joke
+in:iot_hue_lightup
+in:takeaway_order
+in:audio_volume_mute
+in:iot_hue_lightdim
+in:calendar_query
+in:transport_query
+in:transport_taxi
+in:general_greet
+in:music_query
+in:iot_coffee
+in:qa_maths
+in:email_querycontact
+in:recommendation_movies
+in:alarm_remove
+in:calendar_remove
+in:datetime_query
+in:iot_hue_lightchange
+in:iot_wemo_off
+in:transport_ticket
+in:alarm_query
+in:transport_traffic
+in:recommendation_events
+in:lists_createoradd
+in:social_query
+in:social_post
+in:qa_stock
+in:lists_query
+in:qa_factoid
+in:recommendation_locations
+in:audio_volume_other
+in:qa_currency
+in:iot_cleaning
+in:play_audiobook
+in:alarm_set
+in:datetime_convert
+in:play_game
+in:iot_wemo_on
+in:music_dislikeness
+in:email_addcontact
+in:iot_hue_lighton
+in:cooking_query
+in:qa_query
+in:general_negate
+in:general_dontcare
+in:general_repeat
+in:general_affirm
+in:general_commandstop
+in:general_confirm
+in:general_explain
+in:general_praise
+sl:player_setting
+sl:song_name
+sl:house_place
+sl:timeofday
+sl:weather_descriptor
+sl:device_type
+sl:relation
+sl:event_name
+sl:general_frequency
+sl:definition_word
+sl:date
+sl:time
+sl:person
+sl:food_type
+sl:order_type
+sl:business_type
+sl:change_amount
+sl:place_name
+sl:transport_type
+sl:personal_info
+sl:radio_name
+sl:ingredient
+sl:music_genre
+sl:artist_name
+sl:music_descriptor
+sl:playlist_name
+sl:news_topic
+sl:color_type
+sl:podcast_descriptor
+sl:list_name
+sl:media_type
+sl:transport_name
+sl:cooking_type
+sl:joke_type
+sl:currency_name
+sl:email_folder
+sl:app_name
+sl:audiobook_name
+sl:business_name
+sl:meal_type
+sl:podcast_name
+sl:drink_type
+sl:game_name
+sl:transport_agency
+sl:sport_type
+sl:movie_name
+sl:email_address
+sl:transport_descriptor
+sl:alarm_type
+sl:coffee_type
+sl:movie_type
+sl:audiobook_author
+sl:game_type
+sl:music_album
+sl:query_detail
+SEP
+FILL
+in:create_alarm_STOP
+sl:date_time_STOP
+in:get_alarm_STOP
+in:unsupported_alarm_STOP
+in:delete_alarm_STOP
+sl:alarm_name_STOP
+in:get_time_STOP
+sl:date_time_recurring_STOP
+sl:amount_STOP
+sl:duration_STOP
+in:silence_alarm_STOP
+sl:ordinal_STOP
+in:snooze_alarm_STOP
+in:update_alarm_STOP
+sl:period_STOP
+sl:time_zone_STOP
+sl:recurring_date_time_STOP
+in:get_event_STOP
+sl:location_STOP
+in:get_location_STOP
+sl:point_on_map_STOP
+sl:category_event_STOP
+sl:search_radius_STOP
+sl:location_user_STOP
+sl:attribute_event_STOP
+in:get_event_attendee_STOP
+sl:attendee_event_STOP
+sl:location_modifier_STOP
+sl:category_location_STOP
+sl:name_event_STOP
+in:unsupported_event_STOP
+in:get_location_home_STOP
+sl:contact_STOP
+in:get_location_school_STOP
+in:get_contact_STOP
+sl:contact_related_STOP
+sl:type_relation_STOP
+sl:organizer_event_STOP
+in:get_location_work_STOP
+in:negation_STOP
+in:get_event_organizer_STOP
+in:get_event_attendee_amount_STOP
+in:get_message_STOP
+in:send_message_STOP
+sl:recipient_STOP
+sl:content_exact_STOP
+in:react_message_STOP
+sl:type_reaction_STOP
+sl:type_content_STOP
+in:cancel_message_STOP
+sl:group_STOP
+sl:location_home_STOP
+sl:content_emoji_STOP
+sl:type_contact_STOP
+sl:sender_STOP
+sl:mutual_employer_STOP
+sl:resource_STOP
+sl:mutual_school_STOP
+sl:type_info_STOP
+sl:mutual_location_STOP
+sl:tag_message_STOP
+in:unsupported_messaging_STOP
+in:ignore_message_STOP
+in:send_text_message_STOP
+sl:age_STOP
+in:loop_music_STOP
+sl:music_type_STOP
+in:replay_music_STOP
+in:previous_track_music_STOP
+sl:music_track_title_STOP
+in:pause_music_STOP
+sl:music_playlist_title_STOP
+sl:music_provider_name_STOP
+in:skip_track_music_STOP
+sl:music_artist_name_STOP
+in:start_shuffle_music_STOP
+in:dislike_music_STOP
+in:remove_from_playlist_music_STOP
+in:like_music_STOP
+sl:music_album_title_STOP
+in:create_playlist_music_STOP
+in:stop_music_STOP
+in:unsupported_music_STOP
+in:add_to_playlist_music_STOP
+sl:music_radio_id_STOP
+in:get_estimated_duration_STOP
+sl:method_travel_STOP
+sl:source_STOP
+sl:destination_STOP
+in:unsupported_navigation_STOP
+in:get_estimated_arrival_STOP
+sl:date_time_arrival_STOP
+sl:date_time_departure_STOP
+in:get_estimated_departure_STOP
+in:get_info_traffic_STOP
+in:get_directions_STOP
+sl:path_STOP
+in:get_info_road_condition_STOP
+sl:road_condition_STOP
+in:get_distance_STOP
+sl:unit_distance_STOP
+in:update_directions_STOP
+sl:obstruction_avoid_STOP
+sl:path_avoid_STOP
+in:get_info_route_STOP
+sl:road_condition_avoid_STOP
+sl:waypoint_STOP
+sl:location_work_STOP
+sl:waypoint_avoid_STOP
+sl:location_current_STOP
+in:get_location_hometown_STOP
+sl:waypoint_added_STOP
+in:create_reminder_STOP
+sl:person_reminded_STOP
+sl:todo_STOP
+in:get_recurring_date_time_STOP
+sl:frequency_STOP
+in:delete_reminder_STOP
+in:get_todo_STOP
+in:update_reminder_STOP
+in:update_reminder_todo_STOP
+sl:todo_new_STOP
+in:update_reminder_date_time_STOP
+sl:date_time_new_STOP
+in:get_reminder_STOP
+sl:attendee_STOP
+in:reply_message_STOP
+sl:method_retrieval_reminder_STOP
+sl:recurring_date_time_new_STOP
+sl:person_reminded_added_STOP
+in:get_reminder_date_time_STOP
+sl:person_reminded_removed_STOP
+in:get_reminder_location_STOP
+in:get_reminder_amount_STOP
+in:help_reminder_STOP
+sl:name_app_STOP
+sl:attendee_removed_STOP
+sl:attendee_added_STOP
+sl:job_STOP
+in:get_birthday_STOP
+in:create_timer_STOP
+sl:method_timer_STOP
+in:pause_timer_STOP
+in:get_timer_STOP
+in:add_time_timer_STOP
+sl:timer_name_STOP
+in:restart_timer_STOP
+in:subtract_time_timer_STOP
+in:delete_timer_STOP
+in:resume_timer_STOP
+in:unsupported_timer_STOP
+in:update_timer_STOP
+in:get_weather_STOP
+sl:weather_attribute_STOP
+in:get_sunset_STOP
+in:unsupported_weather_STOP
+in:get_sunrise_STOP
+sl:weather_temperature_unit_STOP
+sl:measurement_unit_STOP
+in:get_info_contact_STOP
+in:play_music_STOP
+sl:music_genre_STOP
+<|ner|>
+<|sp|>
+<|STOP|>
+<|SLURP|>
+<|SNIPS|>

exp/asr_train_asr_whisper_full_correct_specaug2_copy_raw_en_whisper_multilingual/RESULTS.md

@@ -0,0 +1,65 @@
+<!-- Generated by scripts/utils/show_asr_result.sh -->
+# RESULTS
+## Environments
+- date: `Sun Jul 16 11:29:20 CDT 2023`
+- python version: `3.9.13 (main, Aug 25 2022, 23:26:10)  [GCC 11.2.0]`
+- espnet version: `espnet 202211`
+- pytorch version: `pytorch 1.12.1+cu116`
+- Git hash: `69261333fd675bacaf981ec33505aec180bfbb38`
+  - Commit date: `Tue Nov 30 16:12:53 2021 -0500`
+
+## asr_train_asr_whisper_full_correct_specaug2_copy_raw_en_whisper_multilingual
+### WER
+
+|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err|
+|---|---|---|---|---|---|---|---|---|
+|decode_asr_accent_rec_asr_model_valid.acc.ave/test_accentdb|3463|3463|99.9|0.1|0.0|0.0|0.1|0.1|
+|decode_asr_ar_asr_model_valid.acc.ave/test_arabic|370|370|100.0|0.0|0.0|0.0|0.0|0.0|
+|decode_asr_auc_asr_model_valid.acc.ave/test_esc50|400|400|67.3|32.8|0.0|0.3|33.0|32.8|
+|decode_asr_er_asr_model_valid.acc.ave/test_iemocap|942|11962|91.0|6.4|2.6|2.5|11.5|63.1|
+|decode_asr_fsd_asr_model_valid.acc.ave/test_asvspoof|71237|71237|98.0|2.0|0.0|0.0|2.0|2.0|
+|decode_asr_gid_asr_model_valid.acc.ave/test_voxceleb|4818|4818|99.9|0.1|0.0|0.0|0.1|0.1|
+|decode_asr_grabo_asr_model_valid.acc.ave/test_grabo|3631|3631|99.7|0.3|0.0|0.0|0.3|0.3|
+|decode_asr_ic1_asr_model_valid.acc.ave/test_snips|166|1470|95.2|2.2|2.6|1.9|6.7|30.1|
+|decode_asr_ic_asr_model_valid.acc.ave/test_fsc|3793|20316|99.8|0.1|0.1|0.1|0.2|0.7|
+|decode_asr_ic_asr_model_valid.acc.ave/test_snips|166|1470|0.0|0.0|100.0|0.0|100.0|100.0|
+|decode_asr_lid_asr_model_valid.acc.ave/test_voxforge|1800|1800|99.9|0.1|0.0|0.0|0.1|0.1|
+|decode_asr_lt_asr_model_valid.acc.ave/test_lt_speech_commands|88|88|98.9|1.1|0.0|0.0|1.1|1.1|
+|decode_asr_ner1_1_asr_model_valid.acc.ave/test_slurp|13078|164549|90.7|4.8|4.5|3.2|12.5|41.7|
+|decode_asr_scd_asr_model_valid.acc.ave/test_mustard|138|138|73.9|26.1|0.0|0.0|26.1|26.1|
+|decode_asr_scd_plus_asr_model_valid.acc.ave/test_mustard_plus_plus|92|92|70.7|29.3|0.0|0.0|29.3|29.3|
+|decode_asr_scr_asr_model_valid.acc.ave/test_speechcommands|4890|4890|99.1|0.9|0.0|0.0|0.9|0.9|
+|decode_asr_sp2_asr_model_valid.acc.ave/test|75636|728701|94.9|1.8|3.3|2.2|7.3|21.6|
+|old_decode_asr_auc_asr_model_valid.acc.ave/test_esc50|400|400|63.0|37.0|0.0|0.0|37.0|37.0|
+|old_decode_asr_scd_asr_model_valid.acc.ave/test_mustard|138|138|73.9|26.1|0.0|0.0|26.1|26.1|
+|old_decode_asr_scd_plus_asr_model_valid.acc.ave/test_mustard_plus_plus|92|92|73.9|26.1|0.0|0.0|26.1|26.1|
+
+### CER
+
+|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err|
+|---|---|---|---|---|---|---|---|---|
+|decode_asr_accent_rec_asr_model_valid.acc.ave/test_accentdb|3463|25897|100.0|0.0|0.0|0.0|0.1|0.1|
+|decode_asr_ar_asr_model_valid.acc.ave/test_arabic|370|370|100.0|0.0|0.0|0.0|0.0|0.0|
+|decode_asr_auc_asr_model_valid.acc.ave/test_esc50|400|528|41.3|48.9|9.8|23.7|82.4|61.0|
+|decode_asr_er_asr_model_valid.acc.ave/test_iemocap|942|58253|95.5|2.1|2.4|2.5|7.0|63.1|
+|decode_asr_fsd_asr_model_valid.acc.ave/test_asvspoof|71237|378250|98.9|0.7|0.4|1.5|2.6|2.0|
+|decode_asr_gid_asr_model_valid.acc.ave/test_voxceleb|4818|4818|99.9|0.1|0.0|0.0|0.1|0.1|
+|decode_asr_grabo_asr_model_valid.acc.ave/test_grabo|3631|168247|99.9|0.0|0.0|0.0|0.1|0.3|
+|decode_asr_ic1_asr_model_valid.acc.ave/test_snips|166|6533|96.4|0.7|2.9|2.3|5.9|30.1|
+|decode_asr_ic_asr_model_valid.acc.ave/test_fsc|3793|172445|99.9|0.0|0.1|0.0|0.1|0.7|
+|decode_asr_ic_asr_model_valid.acc.ave/test_snips|166|6533|0.0|0.0|100.0|0.0|100.0|100.0|
+|decode_asr_lid_asr_model_valid.acc.ave/test_voxforge|1800|3600|99.9|0.1|0.0|0.0|0.1|0.1|
+|decode_asr_lt_asr_model_valid.acc.ave/test_lt_speech_commands|88|526|98.9|0.4|0.8|0.0|1.1|1.1|
+|decode_asr_ner1_1_asr_model_valid.acc.ave/test_slurp|13078|669743|93.5|1.8|4.7|3.2|9.7|41.8|
+|decode_asr_scd_asr_model_valid.acc.ave/test_mustard|138|138|73.9|26.1|0.0|0.0|26.1|26.1|
+|decode_asr_scd_plus_asr_model_valid.acc.ave/test_mustard_plus_plus|92|92|70.7|29.3|0.0|0.0|29.3|29.3|
+|decode_asr_scr_asr_model_valid.acc.ave/test_speechcommands|4890|19959|99.2|0.4|0.4|0.4|1.2|0.9|
+|decode_asr_sp2_asr_model_valid.acc.ave/test|75636|5377833|95.9|0.6|3.5|2.2|6.4|21.6|
+|old_decode_asr_auc_asr_model_valid.acc.ave/test_esc50|400|544|38.2|43.4|18.4|6.6|68.4|58.3|
+|old_decode_asr_scd_asr_model_valid.acc.ave/test_mustard|138|138|73.9|26.1|0.0|0.0|26.1|26.1|
+|old_decode_asr_scd_plus_asr_model_valid.acc.ave/test_mustard_plus_plus|92|92|73.9|26.1|0.0|0.0|26.1|26.1|
+
+### TER
+
+|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err|
+|---|---|---|---|---|---|---|---|---|