init

app.py

@@ -5,6 +5,7 @@ from transformers import (
     CLIPProcessor, CLIPModel, ViltProcessor, ViltForQuestionAnswering
 )
 import torch
+from torch.nn.functional import cosine_similarity
 from PIL import Image
 import functools
 import warnings
@@ -296,6 +297,9 @@ def audio_zero_shot_classifier(audio, candidate_labels, model_name):
     try:
         # Используем CLAP для zero-shot классификации аудио
         from transformers import ClapProcessor, ClapModel
+        import soundfile as sf
+        import numpy as np
+        
         cache_key = f"audio_zero_shot_{model_name}"
         cached = model_cache.get(cache_key)
         if cached is None:
@@ -307,7 +311,27 @@ def audio_zero_shot_classifier(audio, candidate_labels, model_name):
         processor, model = cached
         labels = [label.strip() for label in candidate_labels.split(",")]
         
-        inputs = processor(text=labels, audios=audio, return_tensors="pt", padding=True)
+        # Загружаем аудио файл, если передан путь
+        if isinstance(audio, str):
+            # audio - это путь к файлу
+            audio_data, sample_rate = sf.read(audio)
+        elif isinstance(audio, tuple):
+            # audio - это кортеж (sample_rate, audio_data) от Gradio
+            sample_rate, audio_data = audio
+        else:
+            # audio уже является массивом numpy
+            audio_data = audio
+            sample_rate = None
+        
+        # Преобразуем в numpy array, если нужно
+        if not isinstance(audio_data, np.ndarray):
+            audio_data = np.array(audio_data)
+        
+        # Если аудио моно, убеждаемся что это 1D массив
+        if len(audio_data.shape) > 1:
+            audio_data = audio_data[:, 0] if audio_data.shape[1] > 0 else audio_data.flatten()
+        
+        inputs = processor(text=labels, audios=audio_data, return_tensors="pt", padding=True)
         with torch.no_grad():
             outputs = model(**inputs)
             logits_per_audio = outputs.logits_per_audio
@@ -404,20 +428,75 @@ def image_text_matching(image, text, model_name):
         cache_key = f"clip_{model_name}"
         cached = model_cache.get(cache_key)
         if cached is None:
-            processor = CLIPProcessor.from_pretrained(model_name)
-            model = CLIPModel.from_pretrained(model_name)
-            cached = (processor, model)
+            # Проверяем, является ли модель из sentence-transformers
+            if "sentence-transformers" in model_name:
+                from sentence_transformers import SentenceTransformer
+                model = SentenceTransformer(model_name)
+                cached = ("sentence_transformers", model)
+            # Проверяем, является ли модель LAION (требует OpenCLIP)
+            elif "laion/" in model_name.lower() or "laion5b" in model_name.lower():
+                import open_clip
+                # Определяем имя модели и веса для OpenCLIP
+                if "xlm-roberta-base-ViT-B-32" in model_name or "xlm-roberta-base" in model_name:
+                    clip_model_name = "xlm-roberta-base-ViT-B-32"
+                    pretrained = "laion5b_s13b_b90k"
+                else:
+                    # Пытаемся извлечь информацию из имени модели
+                    clip_model_name = "xlm-roberta-base-ViT-B-32"
+                    pretrained = "laion5b_s13b_b90k"
+                
+                model, _, preprocess = open_clip.create_model_and_transforms(
+                    clip_model_name, 
+                    pretrained=pretrained
+                )
+                tokenizer = open_clip.get_tokenizer(clip_model_name)
+                model.eval()
+                cached = ("openclip", model, preprocess, tokenizer)
+            else:
+                processor = CLIPProcessor.from_pretrained(model_name)
+                model = CLIPModel.from_pretrained(model_name)
+                cached = ("transformers", processor, model)
             model_cache.put(cache_key, cached)
         
-        processor, model = cached
-        inputs = processor(text=[text], images=image, return_tensors="pt", padding=True)
-        
-        with torch.no_grad():
-            outputs = model(**inputs)
-            logits_per_image = outputs.logits_per_image
-            probs = logits_per_image.softmax(dim=1)
+        if cached[0] == "sentence_transformers":
+            # Используем sentence-transformers
+            model = cached[1]
+            # Вычисляем эмбеддинги изображения и текста
+            image_embedding = model.encode(image, convert_to_tensor=True)
+            text_embedding = model.encode(text, convert_to_tensor=True)
+            # Вычисляем косинусное сходство
+            score = cosine_similarity(image_embedding.unsqueeze(0), text_embedding.unsqueeze(0)).item()
+            # Нормализуем в диапазон [0, 1] для лучшей интерпретации
+            score = (score + 1) / 2
+        elif cached[0] == "openclip":
+            # Используем OpenCLIP
+            model, preprocess, tokenizer = cached[1], cached[2], cached[3]
+            # Обрабатываем изображение и текст
+            image_tensor = preprocess(image).unsqueeze(0)
+            text_tokens = tokenizer([text])
+            
+            with torch.no_grad():
+                image_features = model.encode_image(image_tensor)
+                text_features = model.encode_text(text_tokens)
+                # Нормализуем признаки
+                image_features = image_features / image_features.norm(dim=-1, keepdim=True)
+                text_features = text_features / text_features.norm(dim=-1, keepdim=True)
+                # Вычисляем косинусное сходство
+                score = (image_features @ text_features.T).item()
+                # Нормализуем в диапазон [0, 1] для лучшей интерпретации
+                score = (score + 1) / 2
+        else:
+            # Используем стандартный CLIP из transformers
+            processor, model = cached[1], cached[2]
+            inputs = processor(text=[text], images=image, return_tensors="pt", padding=True)
+            
+            with torch.no_grad():
+                outputs = model(**inputs)
+                logits_per_image = outputs.logits_per_image
+                probs = logits_per_image.softmax(dim=1)
+            
+            score = probs[0][0].item()
         
-        score = probs[0][0].item()
         return f"Совпадение изображения и текста: {score:.4f}"
     except Exception as e:
         return f"Ошибка: {str(e)}"
@@ -499,23 +578,87 @@ def image_zero_shot_classification(image, candidate_labels, model_name):
         cache_key = f"clip_zs_{model_name}"
         cached = model_cache.get(cache_key)
         if cached is None:
-            processor = CLIPProcessor.from_pretrained(model_name)
-            model = CLIPModel.from_pretrained(model_name)
-            cached = (processor, model)
+            # Проверяем, является ли модель из sentence-transformers
+            if "sentence-transformers" in model_name:
+                from sentence_transformers import SentenceTransformer
+                model = SentenceTransformer(model_name)
+                cached = ("sentence_transformers", model)
+            # Проверяем, является ли модель LAION (требует OpenCLIP)
+            elif "laion/" in model_name.lower() or "laion5b" in model_name.lower():
+                import open_clip
+                # Определяем имя модели и веса для OpenCLIP
+                if "xlm-roberta-base-ViT-B-32" in model_name or "xlm-roberta-base" in model_name:
+                    clip_model_name = "xlm-roberta-base-ViT-B-32"
+                    pretrained = "laion5b_s13b_b90k"
+                else:
+                    # Пытаемся извлечь информацию из имени модели
+                    clip_model_name = "xlm-roberta-base-ViT-B-32"
+                    pretrained = "laion5b_s13b_b90k"
+                
+                model, _, preprocess = open_clip.create_model_and_transforms(
+                    clip_model_name, 
+                    pretrained=pretrained
+                )
+                tokenizer = open_clip.get_tokenizer(clip_model_name)
+                model.eval()
+                cached = ("openclip", model, preprocess, tokenizer)
+            else:
+                processor = CLIPProcessor.from_pretrained(model_name)
+                model = CLIPModel.from_pretrained(model_name)
+                cached = ("transformers", processor, model)
             model_cache.put(cache_key, cached)
         
-        processor, model = cached
         labels = [label.strip() for label in candidate_labels.split(",")]
-        inputs = processor(text=labels, images=image, return_tensors="pt", padding=True)
         
-        with torch.no_grad():
-            outputs = model(**inputs)
-            logits_per_image = outputs.logits_per_image
-            probs = logits_per_image.softmax(dim=1)
+        if cached[0] == "sentence_transformers":
+            # Используем sentence-transformers
+            model = cached[1]
+            # Вычисляем эмбеддинги изображения и текстов
+            image_embedding = model.encode(image, convert_to_tensor=True)
+            text_embeddings = model.encode(labels, convert_to_tensor=True)
+            # Вычисляем косинусное сходство
+            similarities = cosine_similarity(image_embedding.unsqueeze(0), text_embeddings).squeeze(0)
+            # Нормализуем в диапазон [0, 1] и применяем softmax для вероятностей
+            similarities = (similarities + 1) / 2
+            probs = torch.softmax(similarities, dim=0)
+            
+            output = "Результаты классификации:\n"
+            for label, prob in zip(labels, probs):
+                output += f"{label}: {prob.item():.4f}\n"
+        elif cached[0] == "openclip":
+            # Используем OpenCLIP
+            model, preprocess, tokenizer = cached[1], cached[2], cached[3]
+            # Обрабатываем изображение и тексты
+            image_tensor = preprocess(image).unsqueeze(0)
+            text_tokens = tokenizer(labels)
+            
+            with torch.no_grad():
+                image_features = model.encode_image(image_tensor)
+                text_features = model.encode_text(text_tokens)
+                # Нормализуем признаки
+                image_features = image_features / image_features.norm(dim=-1, keepdim=True)
+                text_features = text_features / text_features.norm(dim=-1, keepdim=True)
+                # Вычисляем косинусное сходство (логиты)
+                logits_per_image = (image_features @ text_features.T) * 100  # Масштабируем для лучшей точности
+                probs = logits_per_image.softmax(dim=1)
+            
+            output = "Результаты классификации:\n"
+            for label, prob in zip(labels, probs[0]):
+                output += f"{label}: {prob.item():.4f}\n"
+        else:
+            # Используем стандартный CLIP из transformers
+            processor, model = cached[1], cached[2]
+            inputs = processor(text=labels, images=image, return_tensors="pt", padding=True)
+            
+            with torch.no_grad():
+                outputs = model(**inputs)
+                logits_per_image = outputs.logits_per_image
+                probs = logits_per_image.softmax(dim=1)
+            
+            output = "Результаты классификации:\n"
+            for label, prob in zip(labels, probs[0]):
+                output += f"{label}: {prob.item():.4f}\n"
         
-        output = "Результаты классификации:\n"
-        for label, prob in zip(labels, probs[0]):
-            output += f"{label}: {prob.item():.4f}\n"
         return output
     except Exception as e:
         return f"Ошибка: {str(e)}"

requirements.txt

@@ -8,3 +8,6 @@ soundfile>=0.12.0
 accelerate>=0.20.0
 sentencepiece>=0.1.99
 datasets>=2.14.0
+timm>=0.9.0
+sentence-transformers>=2.2.0
+open-clip-torch>=2.20.0