import sys sys.stdout.reconfigure(line_buffering=True) try: import spaces except ImportError: # keep @spaces.GPU usable as a no-op; ZeroGPU requires this exact name. class spaces: class GPU: def __init__(self, func=None, duration=60): self.func = func def __call__(self, *args, **kwargs): if self.func is not None: return self.func(*args, **kwargs) func = args[0] return func from types import SimpleNamespace import gradio as gr import librosa import numpy as np import pandas as pd import torch from pyharp import ModelCard, AudioLabel, LabelList, build_endpoint from model.htsat import HTSAT_Swin_Transformer DEVICE = "cuda" if torch.cuda.is_available() else "cpu" # AudioSet configuration values from HTS-AT-repo/config.py CONFIG = SimpleNamespace( sample_rate=32000, window_size=1024, hop_size=320, mel_bins=64, fmin=50, fmax=14000, enable_tscam=True, htsat_attn_heatmap=False, loss_type="clip_bce", enable_repeat_mode=False, ) model = HTSAT_Swin_Transformer(config=CONFIG) checkpoint = torch.load("HTSAT_AudioSet_Saved_1.ckpt", map_location="cpu") state_dict = {key.replace("sed_model.", ""): value for key, value in checkpoint["state_dict"].items()} model.load_state_dict(state_dict) model.eval() model_ready = False # has model been moved onto the device yet? # The model always produces this many framewise steps, regardless of clip length, # by stretching shorter clips onto this fixed grid (so frame i maps linearly onto # real time as i / MODEL_FRAMES * clip_duration). Longer clips instead use a # sliding-window path that averages overlapping crops together, which no longer # maps onto real time at all - framewise localization is only meaningful below # this length until that path is implemented separately. MODEL_FRAMES = model.spec_size * model.freq_ratio labels_df = pd.read_csv("class_label_indice.csv") idx_to_label = dict(zip(labels_df["index"], labels_df["display_name"])) model_card = ModelCard( name="HTS-AT", description=( "Tags the sounds present in an audio clip across 527 AudioSet classes " "(speech, music, instruments, animals, environmental sounds, etc). " "Clips longer than 10 seconds are not currently supported." ), author="Ke Chen, Xingjian Du, Bilei Zhu, Zejun Ma, Taylor Berg-Kirkpatrick, Shlomo Dubnov", tags=["classification", "tagging", "audioset"], ) def find_active_segments(frame_scores: np.ndarray, threshold: float) -> list[tuple[int, int]]: segments = [] start = None for i, score in enumerate(frame_scores): if score > threshold and start is None: start = i elif score <= threshold and start is not None: segments.append((start, i)) start = None if start is not None: segments.append((start, len(frame_scores))) return segments @spaces.GPU @torch.inference_mode() def process_fn(input_audio_path: str, detection_threshold: float): global model_ready if not model_ready: model.to(DEVICE) model_ready = True waveform, _ = librosa.load(input_audio_path, sr=CONFIG.sample_rate) duration = len(waveform) / CONFIG.sample_rate audio_tensor = torch.from_numpy(waveform).float().to(DEVICE)[None, :] mel_frames = model.logmel_extractor(model.spectrogram_extractor(audio_tensor)).shape[2] if mel_frames > MODEL_FRAMES: raise gr.Error( f"Clip is too long ({duration:.1f}s). This model currently supports " f"clips up to about {MODEL_FRAMES * CONFIG.hop_size / CONFIG.sample_rate:.1f}s." ) framewise_output = model(audio_tensor, None, False)["framewise_output"][0].cpu().numpy() label_list = LabelList() for class_idx in range(framewise_output.shape[1]): frame_scores = framewise_output[:, class_idx] for start, end in find_active_segments(frame_scores, detection_threshold): peak_confidence = frame_scores[start:end].max() label_list.append(AudioLabel( t=start / MODEL_FRAMES * duration, duration=(end - start) / MODEL_FRAMES * duration, label=idx_to_label[class_idx], description=f"confidence {peak_confidence:.0%}", amplitude=float(peak_confidence), )) return input_audio_path, label_list with gr.Blocks() as demo: input_components = [ gr.Audio(type="filepath", label="Input Audio").harp_required(True), gr.Slider(minimum=0.0, maximum=1.0, step=0.05, value=0.5, label="Detection Threshold", info="Minimum confidence for a sound to be tagged"), ] output_components = [ gr.Audio(type="filepath", label="Output Audio").set_info("Input audio, unchanged."), gr.JSON(label="Detected Sounds").set_info("Top predicted sound tags with confidence scores."), ] build_endpoint( model_card=model_card, input_components=input_components, output_components=output_components, process_fn=process_fn, ) demo.queue().launch(pwa=True)