init

app.py

@@ -303,20 +303,9 @@ def audio_classifier(audio, model_name):
 def audio_zero_shot_classifier(audio, candidate_labels, model_name):
     """Zero-shot классификация аудио"""
     try:
-        # Используем CLAP для zero-shot классификации аудио
-        from transformers import ClapProcessor, ClapModel
         import soundfile as sf
         import numpy as np
         
-        cache_key = f"audio_zero_shot_{model_name}"
-        cached = model_cache.get(cache_key)
-        if cached is None:
-            processor = ClapProcessor.from_pretrained(model_name)
-            model = ClapModel.from_pretrained(model_name)
-            cached = (processor, model)
-            model_cache.put(cache_key, cached)
-        
-        processor, model = cached
         labels = [label.strip() for label in candidate_labels.split(",")]
         
         # Загружаем аудио файл, если передан путь
@@ -339,16 +328,99 @@ def audio_zero_shot_classifier(audio, candidate_labels, model_name):
         if len(audio_data.shape) > 1:
             audio_data = audio_data[:, 0] if audio_data.shape[1] > 0 else audio_data.flatten()
         
-        inputs = processor(text=labels, audios=audio_data, return_tensors="pt", padding=True)
-        with torch.no_grad():
-            outputs = model(**inputs)
-            logits_per_audio = outputs.logits_per_audio
-            probs = logits_per_audio.softmax(dim=1)
-        
-        output = "Результаты классификации:\n"
-        for label, prob in zip(labels, probs[0]):
-            output += f"{label}: {prob.item():.4f}\n"
-        return output
+        # Проверяем тип модели
+        if "clap" in model_name.lower():
+            # Используем CLAP для zero-shot классификации аудио
+            from transformers import ClapProcessor, ClapModel
+            
+            cache_key = f"audio_zero_shot_{model_name}"
+            cached = model_cache.get(cache_key)
+            if cached is None:
+                processor = ClapProcessor.from_pretrained(model_name)
+                model = ClapModel.from_pretrained(model_name)
+                cached = (processor, model)
+                model_cache.put(cache_key, cached)
+            
+            processor, model = cached
+            inputs = processor(text=labels, audios=audio_data, return_tensors="pt", padding=True)
+            with torch.no_grad():
+                outputs = model(**inputs)
+                logits_per_audio = outputs.logits_per_audio
+                probs = logits_per_audio.softmax(dim=1)
+            
+            output = "Результаты классификации:\n"
+            for label, prob in zip(labels, probs[0]):
+                output += f"{label}: {prob.item():.4f}\n"
+            return output
+        elif "wav2vec2" in model_name.lower() or "hubert" in model_name.lower():
+            # Используем подход с audio embeddings + text embeddings
+            # Получаем аудио эмбеддинги через audio model и текстовые эмбеддинги через text model
+            from transformers import AutoProcessor, AutoModel
+            from sentence_transformers import SentenceTransformer
+            
+            cache_key = f"audio_zero_shot_{model_name}"
+            cached = model_cache.get(cache_key)
+            if cached is None:
+                # Загружаем модель для аудио эмбеддингов
+                audio_processor = AutoProcessor.from_pretrained(model_name)
+                audio_model = AutoModel.from_pretrained(model_name)
+                # Загружаем модель для текстовых эмбеддингов
+                text_model = SentenceTransformer('all-MiniLM-L6-v2')
+                cached = (audio_processor, audio_model, text_model)
+                model_cache.put(cache_key, cached)
+            
+            audio_processor, audio_model, text_model = cached
+            
+            # Получаем аудио эмбеддинги
+            # Нормализуем sample rate если нужно
+            if sample_rate is None:
+                sample_rate = 16000
+            inputs = audio_processor(audio_data, sampling_rate=sample_rate, return_tensors="pt")
+            with torch.no_grad():
+                audio_outputs = audio_model(**inputs)
+                # Используем последний скрытый слой как эмбеддинг
+                if hasattr(audio_outputs, 'last_hidden_state'):
+                    audio_embedding = audio_outputs.last_hidden_state.mean(dim=1)  # Усредняем по временной оси
+                else:
+                    audio_embedding = audio_outputs[0].mean(dim=1)
+                audio_embedding = audio_embedding / audio_embedding.norm(dim=1, keepdim=True)
+            
+            # Получаем текстовые эмбеддинги
+            text_embeddings = text_model.encode(labels, convert_to_tensor=True)
+            text_embeddings = text_embeddings / text_embeddings.norm(dim=1, keepdim=True)
+            
+            # Вычисляем косинусное сходство
+            similarities = cosine_similarity(audio_embedding, text_embeddings).squeeze(0)
+            # Применяем softmax для получения вероятностей
+            probs = torch.softmax(similarities * 10, dim=0)  # Масштабируем для лучшей точности
+            
+            output = "Результаты классификации (через audio + text embeddings):\n"
+            for label, prob in zip(labels, probs):
+                output += f"{label}: {prob.item():.4f}\n"
+            return output
+        else:
+            # Для других моделей используем CLAP по умолчанию
+            from transformers import ClapProcessor, ClapModel
+            
+            cache_key = f"audio_zero_shot_{model_name}"
+            cached = model_cache.get(cache_key)
+            if cached is None:
+                processor = ClapProcessor.from_pretrained(model_name)
+                model = ClapModel.from_pretrained(model_name)
+                cached = (processor, model)
+                model_cache.put(cache_key, cached)
+            
+            processor, model = cached
+            inputs = processor(text=labels, audios=audio_data, return_tensors="pt", padding=True)
+            with torch.no_grad():
+                outputs = model(**inputs)
+                logits_per_audio = outputs.logits_per_audio
+                probs = logits_per_audio.softmax(dim=1)
+            
+            output = "Результаты классификации:\n"
+            for label, prob in zip(labels, probs[0]):
+                output += f"{label}: {prob.item():.4f}\n"
+            return output
     except Exception as e:
         return f"Ошибка: {str(e)}"
 
@@ -1258,7 +1330,11 @@ with gr.Blocks(title="Трансформеры Hugging Face", theme=gr.themes.So
                             value="music, speech, noise"
                         )
                         zs_audio_model = gr.Dropdown(
-                            choices=["laion/clap-htsat-unfused"],
+                            choices=[
+                                "laion/clap-htsat-unfused",
+                                "laion/clap-htsat-fused",
+                                "facebook/wav2vec2-base-960h"
+                            ],
                             value="laion/clap-htsat-unfused",
                             label="Выберите модель"
                         )
@@ -1307,10 +1383,11 @@ with gr.Blocks(title="Трансформеры Hugging Face", theme=gr.themes.So
                         )
                         tts_model = gr.Dropdown(
                             choices=[
-                                "microsoft/speecht5_tts"
+                                "microsoft/speecht5_tts",
+                                "facebook/mms-tts-eng"
                             ],
                             value="microsoft/speecht5_tts",
-                            label="Выберите модель (поддерживаются только модели SpeechT5 из transformers)"
+                            label="Выберите модель"
                         )
                         tts_btn = gr.Button("Синтезировать", variant="primary")
                     with gr.Column():