| --- |
| datasets: |
| - as2m |
| license: cc-by-4.0 |
| pipeline_tag: audio-classification |
| library_name: espnet |
| tags: |
| - espnet |
| - audio |
| - classification |
| --- |
| |
| ## ESPnet2 CLS model |
|
|
| ### `espnet/OpenBEATS-Base-i3-as2m` |
|
|
| This model was trained by Shikhar Bharadwaj using as2m recipe in [espnet](https://github.com/espnet/espnet/). |
|
|
| It is presented in the paper [OpenBEATs: A Fully Open-Source General-Purpose Audio Encoder](https://huggingface.co/papers/2507.14129). |
|
|
| * **GitHub Repository:** [Shikhar-S/OpenBEATs](https://github.com/Shikhar-S/OpenBEATs) |
|
|
| ### Usage |
|
|
| You can run this model using the `openbeats` library: |
|
|
| ```bash |
| pip install openbeats |
| ``` |
|
|
| ```python |
| from openbeats.model import OpenBeats |
| from openbeats.utils import load_audio |
| |
| # load model |
| model = OpenBeats.from_pretrained("espnet/OpenBEATS-Base-i3-as2m", device="cuda") |
| |
| # from a file with any sample rate |
| out = model.encode_file("audio.wav") # pass chunk_seconds=10 for long audio |
| |
| # or load the waveform in 16khz monoaural array with values in [-1,1] |
| wav, sr = load_audio("audio.wav") |
| # and pass it |
| out = model.encode(wav, sr) |
| |
| print(out["patch_embeddings"].shape) |
| ``` |
|
|
| ## CLS config |
|
|
| <details><summary>expand</summary> |
|
|
| ``` |
| config: /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/earbasei3/conf/ear_base/audioset2m.yaml |
| print_config: false |
| log_level: INFO |
| drop_last_iter: false |
| dry_run: false |
| iterator_type: sequence |
| valid_iterator_type: null |
| output_dir: /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/audioset2m/cls_earbasei3 |
| ngpu: 0 |
| seed: 0 |
| num_workers: 2 |
| num_att_plot: 0 |
| dist_backend: nccl |
| dist_init_method: env:// |
| dist_world_size: null |
| dist_rank: null |
| local_rank: null |
| dist_master_addr: null |
| dist_master_port: null |
| dist_launcher: null |
| multiprocessing_distributed: false |
| unused_parameters: true |
| sharded_ddp: false |
| use_deepspeed: false |
| deepspeed_config: null |
| gradient_as_bucket_view: true |
| ddp_comm_hook: null |
| cudnn_enabled: true |
| cudnn_benchmark: false |
| cudnn_deterministic: true |
| use_tf32: false |
| collect_stats: false |
| write_collected_feats: false |
| max_epoch: 16 |
| patience: null |
| val_scheduler_criterion: |
| - valid |
| - loss |
| early_stopping_criterion: |
| - valid |
| - loss |
| - min |
| best_model_criterion: |
| - - valid |
| - epoch_mAP |
| - max |
| keep_nbest_models: 1 |
| nbest_averaging_interval: 0 |
| grad_clip: 1 |
| grad_clip_type: 2.0 |
| grad_noise: false |
| accum_grad: 2 |
| no_forward_run: false |
| resume: true |
| train_dtype: float32 |
| use_amp: false |
| log_interval: null |
| use_matplotlib: true |
| use_tensorboard: true |
| create_graph_in_tensorboard: false |
| use_wandb: true |
| wandb_project: audioverse |
| wandb_id: null |
| wandb_entity: shikhar |
| wandb_name: audioset2m.earbasei3 |
| wandb_model_log_interval: -1 |
| detect_anomaly: false |
| use_adapter: false |
| adapter: lora |
| save_strategy: all |
| adapter_conf: {} |
| pretrain_path: null |
| init_param: [] |
| ignore_init_mismatch: false |
| freeze_param: [] |
| num_iters_per_epoch: null |
| batch_size: 320 |
| valid_batch_size: 1200 |
| batch_bins: 1000000 |
| valid_batch_bins: null |
| category_sample_size: 10 |
| train_shape_file: |
| - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/audioset2m/cls_stats_16k/train/speech_shape |
| - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/audioset2m/cls_stats_16k/train/label_shape |
| valid_shape_file: |
| - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/audioset2m/cls_stats_16k/valid/speech_shape |
| - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/audioset2m/cls_stats_16k/valid/label_shape |
| batch_type: folded |
| valid_batch_type: folded |
| fold_length: |
| - 160000 |
| - 600 |
| sort_in_batch: descending |
| shuffle_within_batch: false |
| sort_batch: descending |
| multiple_iterator: false |
| utt2weight_file: null |
| chunk_length: 500 |
| chunk_shift_ratio: 0.5 |
| num_cache_chunks: 1024 |
| chunk_excluded_key_prefixes: [] |
| chunk_default_fs: null |
| chunk_max_abs_length: null |
| chunk_discard_short_samples: true |
| train_data_path_and_name_and_type: |
| - - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/dump/audioset2m/train/wav.scp |
| - speech |
| - sound |
| - - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/dump/audioset2m/train/text |
| - label |
| - text |
| valid_data_path_and_name_and_type: |
| - - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/dump/audioset2m/eval/wav.scp |
| - speech |
| - sound |
| - - /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/dump/audioset2m/eval/text |
| - label |
| - text |
| multi_task_dataset: false |
| allow_variable_data_keys: false |
| max_cache_size: 0.0 |
| max_cache_fd: 32 |
| allow_multi_rates: false |
| valid_max_cache_size: null |
| exclude_weight_decay: false |
| exclude_weight_decay_conf: {} |
| optim: adamw |
| optim_conf: |
| lr: 0.0001 |
| weight_decay: 0.01 |
| betas: |
| - 0.9 |
| - 0.98 |
| scheduler: cosineannealingwarmuprestarts |
| scheduler_conf: |
| first_cycle_steps: 50000 |
| warmup_steps: 5000 |
| max_lr: 0.0001 |
| min_lr: 5.0e-06 |
| lightning_conf: |
| log_every_n_steps: 250 |
| max_epochs: 16 |
| strategy: ddp |
| strategy_conf: |
| find_unused_parameters: true |
| best_model_criterion: |
| - - valid/epoch_mAP |
| - max |
| - 1 |
| devices: 4 |
| num_nodes: 1 |
| default_root_dir: /work/nvme/bbjs/sbharadwaj/espnet/egs2/audioverse/v1/exp/audioset2m/cls_earbasei3 |
| token_list: |
| - Music |
| - Speech |
| - Vehicle |
| - Musical_instrument |
| - Inside,_small_room |
| - Guitar |
| - Plucked_string_instrument |
| - Singing |
| - Car |
| - Animal |
| - Electronic_music |
| - Outside,_rural_or_natural |
| - Outside,_urban_or_manmade |
| - Violin,_fiddle |
| - Inside,_large_room_or_hall |
| - Bird |
| - Drum |
| - Domestic_animals,_pets |
| - Dubstep |
| - Male_speech,_man_speaking |
| - Techno |
| - Percussion |
| - Engine |
| - Narration,_monologue |
| - Drum_kit |
| - Acoustic_guitar |
| - Strum |
| - Dog |
| - Boat,_Water_vehicle |
| - Train |
| - Electric_guitar |
| - Accelerating,_revving,_vroom |
| - Piano |
| - Child_speech,_kid_speaking |
| - Truck |
| - Keyboard_(musical) |
| - Crowd |
| - Bowed_string_instrument |
| - Bass_drum |
| - Rock_and_roll |
| - Motor_vehicle_(road) |
| - Pop_music |
| - Rail_transport |
| - Pigeon,_dove |
| - Water |
| - Female_speech,_woman_speaking |
| - Orchestra |
| - Rock_music |
| - Sound_effect |
| - Motorboat,_speedboat |
| - Railroad_car,_train_wagon |
| - Siren |
| - Tools |
| - Female_singing |
| - Hip_hop_music |
| - Silence |
| - Motorcycle |
| - Male_singing |
| - Brass_instrument |
| - Classical_music |
| - Snare_drum |
| - Inside,_public_space |
| - Choir |
| - Soundtrack_music |
| - House_music |
| - Wind |
| - Race_car,_auto_racing |
| - Heavy_metal |
| - Bass_guitar |
| - Chicken,_rooster |
| - Wind_noise_(microphone) |
| - Idling |
| - Medium_engine_(mid_frequency) |
| - Bird_vocalization,_bird_call,_bird_song |
| - Fowl |
| - Wind_instrument,_woodwind_instrument |
| - Exciting_music |
| - Country |
| - Laughter |
| - Marimba,_xylophone |
| - Sampler |
| - Emergency_vehicle |
| - Aircraft |
| - Electronica |
| - Ukulele |
| - Cello |
| - Bus |
| - Rhythm_and_blues |
| - Synthesizer |
| - Background_music |
| - Jazz |
| - Mantra |
| - Dance_music |
| - Flute |
| - Blues |
| - Effects_unit |
| - Electric_piano |
| - Cymbal |
| - Chirp,_tweet |
| - Rimshot |
| - Rapping |
| - Trance_music |
| - Livestock,_farm_animals,_working_animals |
| - Run |
| - Cheering |
| - Electronic_dance_music |
| - Theme_music |
| - Gospel_music |
| - Heavy_engine_(low_frequency) |
| - Radio |
| - Music_of_Latin_America |
| - Disco |
| - Glockenspiel |
| - Tender_music |
| - Punk_rock |
| - Funk |
| - Gunshot,_gunfire |
| - Hi-hat |
| - Bow-wow |
| - Cat |
| - Music_of_Asia |
| - Trumpet |
| - Car_passing_by |
| - Drum_and_bass |
| - Vehicle_horn,_car_horn,_honking |
| - Helicopter |
| - Vocal_music |
| - Police_car_(siren) |
| - Pizzicato |
| - Progressive_rock |
| - Tap |
| - Printer |
| - Video_game_music |
| - Music_for_children |
| - Clip-clop |
| - Ocean |
| - Drum_machine |
| - Rain |
| - Horse |
| - Power_tool |
| - Wood |
| - Lullaby |
| - Mallet_percussion |
| - Reggae |
| - Fire_engine,_fire_truck_(siren) |
| - New-age_music |
| - Christian_music |
| - Spray |
| - Saxophone |
| - Fireworks |
| - Skateboard |
| - Independent_music |
| - Fixed-wing_aircraft,_airplane |
| - Drum_roll |
| - Insect |
| - Bicycle |
| - Coo |
| - Tick-tock |
| - Accordion |
| - Scratching_(performance_technique) |
| - Soul_music |
| - Rain_on_surface |
| - Stream |
| - Rowboat,_canoe,_kayak |
| - Organ |
| - Ambient_music |
| - Steel_guitar,_slide_guitar |
| - Waves,_surf |
| - Distortion |
| - Music_of_Africa |
| - Trombone |
| - Bluegrass |
| - Sailboat,_sailing_ship |
| - Afrobeat |
| - Sheep |
| - Child_singing |
| - Bark |
| - Duck |
| - Salsa_music |
| - Quack |
| - Door |
| - Hiss |
| - Cluck |
| - Beatboxing |
| - Music_of_Bollywood |
| - Banjo |
| - Vibration |
| - Water_tap,_faucet |
| - Baby_cry,_infant_cry |
| - Mandolin |
| - Yip |
| - Flamenco |
| - Snoring |
| - Psychedelic_rock |
| - Opera |
| - Explosion |
| - Double_bass |
| - Burst,_pop |
| - Toilet_flush |
| - Conversation |
| - Applause |
| - Television |
| - Harmonica |
| - Train_horn |
| - Folk_music |
| - Wood_block |
| - Christmas_music |
| - Clarinet |
| - Steam |
| - Typing |
| - Crowing,_cock-a-doodle-doo |
| - Bleat |
| - Basketball_bounce |
| - Middle_Eastern_music |
| - Goat |
| - Roll |
| - Harp |
| - Harpsichord |
| - Computer_keyboard |
| - Grunge |
| - Vacuum_cleaner |
| - Whistling |
| - Drill |
| - Clickety-clack |
| - Meow |
| - Chatter |
| - Gurgling |
| - A_capella |
| - Civil_defense_siren |
| - Whoop |
| - Sewing_machine |
| - Bee,_wasp,_etc. |
| - Ambulance_(siren) |
| - Machine_gun |
| - Bell |
| - Honk |
| - Snicker |
| - Whimper_(dog) |
| - Dishes,_pots,_and_pans |
| - Tapping_(guitar_technique) |
| - White_noise |
| - Sad_music |
| - Thump,_thud |
| - Chainsaw |
| - Goose |
| - Bagpipes |
| - Oink |
| - Tick |
| - Song |
| - Traditional_music |
| - Tabla |
| - Scary_music |
| - Subway,_metro,_underground |
| - Rustling_leaves |
| - Liquid |
| - Ska |
| - Chant |
| - Speech_synthesizer |
| - Canidae,_dogs,_wolves |
| - Walk,_footsteps |
| - Mechanisms |
| - Chuckle,_chortle |
| - Traffic_noise,_roadway_noise |
| - Bathtub_(filling_or_washing) |
| - Boom |
| - Carnatic_music |
| - Fusillade |
| - Swing_music |
| - Whispering |
| - Lawn_mower |
| - Crumpling,_crinkling |
| - Frying_(food) |
| - Beep,_bleep |
| - Sitar |
| - Tire_squeal |
| - Whack,_thwack |
| - Sink_(filling_or_washing) |
| - Skidding |
| - Vibraphone |
| - Rub |
| - Waterfall |
| - Crackle |
| - Rustle |
| - Smash,_crash |
| - Hubbub,_speech_noise,_speech_babble |
| - Happy_music |
| - Electronic_tuner |
| - Crying,_sobbing |
| - Fire |
| - Jingle_(music) |
| - Fly,_housefly |
| - Hammond_organ |
| - Whoosh,_swoosh,_swish |
| - Hum |
| - Timpani |
| - Shout |
| - Clatter |
| - Trickle,_dribble |
| - Sizzle |
| - Electronic_organ |
| - Burping,_eructation |
| - Firecracker |
| - Throbbing |
| - Telephone |
| - Thunder |
| - Echo |
| - Blender |
| - Thunderstorm |
| - Frog |
| - Whimper |
| - Didgeridoo |
| - Environmental_noise |
| - Snake |
| - Raindrop |
| - Fart |
| - French_horn |
| - Slosh |
| - Zither |
| - Screaming |
| - Jingle,_tinkle |
| - Church_bell |
| - String_section |
| - Sneeze |
| - Jingle_bell |
| - Aircraft_engine |
| - Funny_music |
| - Angry_music |
| - Wild_animals |
| - Turkey |
| - Engine_starting |
| - Rattle |
| - Arrow |
| - Ringtone |
| - Propeller,_airscrew |
| - Sliding_door |
| - Heart_sounds,_heartbeat |
| - Pink_noise |
| - Steelpan |
| - Giggle |
| - Pig |
| - Buzzer |
| - Artillery_fire |
| - Splash,_splatter |
| - Roar |
| - Fire_alarm |
| - Hiccup |
| - Gobble |
| - Air_brake |
| - Chime |
| - Plop |
| - Singing_bowl |
| - Cattle,_bovinae |
| - Cutlery,_silverware |
| - Slap,_smack |
| - Cough |
| - Ship |
| - Reverberation |
| - Babbling |
| - Cacophony |
| - Electric_shaver,_electric_razor |
| - Baby_laughter |
| - Cricket |
| - Crow |
| - Writing |
| - Glass |
| - Howl |
| - Slam |
| - Belly_laugh |
| - Mechanical_fan |
| - Breathing |
| - Children_playing |
| - Chewing,_mastication |
| - Microwave_oven |
| - Stir |
| - Wedding_music |
| - Ping |
| - Sawing |
| - Eruption |
| - Steam_whistle |
| - Yell |
| - Clapping |
| - Alarm |
| - Chink,_clink |
| - Growling |
| - Roaring_cats_(lions,_tigers) |
| - Drip |
| - Air_horn,_truck_horn |
| - Clicking |
| - Toot |
| - Ding |
| - Harmonic |
| - Jet_engine |
| - Moo |
| - Tambourine |
| - Train_whistle |
| - Clock |
| - Children_shouting |
| - Engine_knocking |
| - Patter |
| - Ratchet,_pawl |
| - Filing_(rasp) |
| - Groan |
| - Telephone_bell_ringing |
| - Change_ringing_(campanology) |
| - Cap_gun |
| - Whistle |
| - Synthetic_singing |
| - Rodents,_rats,_mice |
| - Fill_(with_liquid) |
| - Owl |
| - Theremin |
| - Typewriter |
| - Gears |
| - Pump_(liquid) |
| - Caw |
| - Maraca |
| - Neigh,_whinny |
| - Alarm_clock |
| - Wind_chime |
| - Shuffle |
| - Smoke_detector,_smoke_alarm |
| - Train_wheels_squealing |
| - Purr |
| - Static |
| - Caterwaul |
| - Snort |
| - Bellow |
| - Tubular_bells |
| - Yodeling |
| - Keys_jangling |
| - Battle_cry |
| - Hammer |
| - Rumble |
| - Gong |
| - Single-lens_reflex_camera |
| - Flap |
| - Boing |
| - Car_alarm |
| - Ding-dong |
| - Breaking |
| - Sine_wave |
| - Telephone_dialing,_DTMF |
| - Whip |
| - Boiling |
| - Buzz |
| - Mains_hum |
| - Chop |
| - Heart_murmur |
| - Light_engine_(high_frequency) |
| - Hands |
| - Camera |
| - Humming |
| - Thunk |
| - Rattle_(instrument) |
| - Scrape |
| - Gush |
| - Mouse |
| - Air_conditioning |
| - Scratch |
| - Pour |
| - Coin_(dropping) |
| - Hair_dryer |
| - Chorus_effect |
| - Croak |
| - Squish |
| - Dial_tone |
| - Crunch |
| - Tearing |
| - Field_recording |
| - Cash_register |
| - Zipper_(clothing) |
| - Bang |
| - Shatter |
| - Biting |
| - Throat_clearing |
| - Bird_flight,_flapping_wings |
| - Noise |
| - Mosquito |
| - Reversing_beeps |
| - Sonar |
| - Grunt |
| - Shofar |
| - Doorbell |
| - Ice_cream_truck,_ice_cream_van |
| - Gasp |
| - Shuffling_cards |
| - Knock |
| - Sigh |
| - Scissors |
| - Clang |
| - Cowbell |
| - Pant |
| - Electric_toothbrush |
| - Jackhammer |
| - Busy_signal |
| - Drawer_open_or_close |
| - Whir |
| - Power_windows,_electric_windows |
| - Crack |
| - Whale_vocalization |
| - Zing |
| - Stomach_rumble |
| - Wail,_moan |
| - Bouncing |
| - Pulse |
| - Foghorn |
| - Bicycle_bell |
| - Sniff |
| - Chirp_tone |
| - Squeak |
| - Cupboard_open_or_close |
| - Sanding |
| - Sidetone |
| - Wheeze |
| - Squawk |
| - Squeal |
| - Splinter |
| - Dental_drill,_dentist's_drill |
| - Finger_snapping |
| - Chopping_(food) |
| - Tuning_fork |
| - Gargling |
| - Pulleys |
| - Toothbrush |
| - Creak |
| - Crushing |
| - Hoot |
| - <blank> |
| - <unk> |
| text_token_list: null |
| text_bpemodel: null |
| init: xavier_normal |
| input_size: 1 |
| use_preprocessor: true |
| frontend: null |
| frontend_conf: {} |
| specaug: null |
| specaug_conf: {} |
| normalize: null |
| normalize_conf: {} |
| preencoder: null |
| preencoder_conf: {} |
| encoder: beats |
| encoder_conf: |
| beats_ckpt_path: /work/nvme/bbjs/sbharadwaj/7Msounds/exp/beats_iter2_base2.tune_lr5e-4_warmup40000_bins1600000_totalsteps400000/epoch_latest.pt |
| beats_config: |
| layer_wise_gradient_decay_ratio: 0.6 |
| encoder_layerdrop: 0.1 |
| dropout: 0.0 |
| use_weighted_representation: false |
| specaug_config: |
| apply_time_warp: true |
| apply_freq_mask: false |
| apply_time_mask: true |
| time_mask_width_ratio_range: |
| - 0 |
| - 0.06 |
| num_time_mask: 1 |
| roll_augment: true |
| roll_interval: 1 |
| text_encoder: null |
| text_encoder_conf: {} |
| embedding_fusion: null |
| embedding_fusion_conf: {} |
| decoder: linear |
| decoder_conf: {} |
| model: espnet |
| model_conf: |
| classification_type: multi-label |
| mixup_probability: 0.8 |
| lsm_weight: 0.0 |
| log_epoch_metrics: true |
| user_callbacks: |
| - mAP_logging |
| required: |
| - output_dir |
| - token_list |
| task: cls |
| ``` |
|
|
| </details> |
|
|
| ### Citations |
|
|
| ```BibTex |
| |
| @article{bharadwaj2025openbeats, |
| title={OpenBEATs: A Fully Open-Source General-Purpose Audio Encoder}, |
| author={Bharadwaj, Shikhar and Cornell, Samuele and Choi, Kwanghee and Fukayama, Satoru and Shim, Hye-jin and Deshmukh, Soham and Watanabe, Shinji}, |
| journal={arXiv preprint arXiv:2507.14129}, |
| year={2025} |
| } |
| |
| @inproceedings{watanabe2018espnet, |
| author={Shinji Watanabe and Takaaki Hori and Shigeki Karita and Tomoki Hayashi and Jiro Nishitoba and Yuya Unno and Nelson Yalta and Jahn Heymann and Matthew Wiesner and Nanxin Chen and Adithya Renduchintala and Tsubasa Ochiai}, |
| title={{ESPnet}: End-to-End Speech Processing Toolkit}, |
| year={2018}, |
| booktitle={Proceedings of Interspeech}, |
| pages={2207--2211}, |
| doi={10.21437/Interspeech.2018-1456}, |
| url={http://dx.doi.org/10.21437/Interspeech.2018-1456} |
| } |
| |
| |
| |
| |
| |
| |
| ``` |
