Upload 5 files

app.py

@@ -0,0 +1,67 @@
+import gradio as gr
+import plotly.express as px
+import pandas as pd
+import logging
+import torch
+import numpy as np
+import pandas as pd
+
+from inference import predict
+
+logging.basicConfig(level=logging.INFO)
+
+    
+def plotly_plot_video(video_path):
+    data = pd.DataFrame()
+    data['Emotion'] = ['😠 anger', '🤢 disgust', '😨 fear', '😄 joy/happiness', '😐 neutral', '😢 sadness', '😲 surprise/enthusiasm']
+    try:
+        data['Probability'] = predict(video_path)[0]
+        p = px.bar(data, x='Emotion', y='Probability', color="Probability")
+        return (
+            p,
+            f"## ✔️ Dominant Emotion: {data['Emotion'].values[np.argmax(np.array(data['Probability']))]}"
+        )
+
+    except Exception as e:
+        logging.error(f"Processing failed: {e}")
+        data['Probability'] = [0] * data.shape[0]
+        p = px.bar(data, x='Emotion', y='Probability', color="Probability")
+        return (
+            p,
+            "⚠️ Processing Error"
+        )
+
+def create_demo_video():
+    with gr.Blocks(theme='Nymbo/rounded-gradient', css=".gradio-container {background-color: #F0F8FF}", title="Emotion Detection") as demo:
+        gr.Markdown("# Мультимодальная модель")
+
+        with gr.Row():
+            video_input = gr.Video(
+                sources=["upload", "webcam"],
+                type="filepath",
+                label="Record or Upload Video",
+                format="mp4",
+                interactive=True
+            )
+        with gr.Row():
+            top_emotion = gr.Markdown("## ✔️ Dominant Emotion: Waiting for input ...",
+                                      elem_classes="dominant-emotion")
+
+        with gr.Row():
+            text_plot = gr.Plot(label="Text Analysis")
+
+        video_input.change(fn=plotly_plot_video, inputs=video_input, outputs=[text_plot, top_emotion])
+    return demo
+
+def create_demo():
+    audio = create_demo_video()
+    demo = gr.TabbedInterface(
+        [audio],
+        ["Video Prediction"],
+    )
+    return demo
+    
+
+if __name__ == "__main__":
+    demo = create_demo()
+    demo.launch()

config_copy.toml

@@ -0,0 +1,118 @@
+# ─────────────────────────────
+# Dataset Settings
+# ─────────────────────────────
+
+[datasets.cmu_mosei]
+base_dir = "../data/CMU-MOSEI"
+csv_path = "{base_dir}/{split}_full.csv"
+video_dir = "{base_dir}/video/{split}/"
+
+[datasets.fiv2]
+base_dir = "../data/FirstImpressionsV2"
+csv_path = "{base_dir}/{split}_full.csv"
+video_dir = "{base_dir}/video/{split}/"
+
+
+# ─────────────────────────────
+# DataLoader Parameters
+# ─────────────────────────────
+[dataloader]
+num_workers = 0
+shuffle = true
+prepare_only = false
+average_features = true
+
+
+# ─────────────────────────────
+# General Training Parameters
+# ─────────────────────────────
+[train.general]
+random_seed = 42                # fixed random seed (0 = random each run)
+subset_size = 0                 # subset limit (0 = use entire dataset)
+batch_size = 32                 # batch size
+num_epochs = 100                # total number of training epochs
+max_patience = 25               # max epochs without improvement (Early Stopping)
+save_best_model = true          # save the best model
+save_prepared_data = true       # save precomputed embeddings
+save_feature_path = "../../../../s3_ml_data/dokoryakovskaya/train/train/features/" # path to save embeddings
+search_type = "greedy"      # "greedy", "exhaustive", or "none"
+checkpoint_dir = "checkpoints"
+device = "cuda"                 # "cuda" or "cpu"
+selection_metric = "mean_combo" # metric for model selection: mean_combo, mean_emo, mF1, mUAR, ACC, etc.
+single_task = true
+opt_set = 'test'
+
+# ─────────────────────────────
+# Model Parameters
+# ─────────────────────────────
+[train.model]
+id_ablation_type_by_modality  = 0
+id_ablation_type_by_component = 6
+single_task_id = 0
+model_stage = "fusion"              # stage: personality, emotion, fusion
+per_activation = "relu"             # activation for personality branch
+hidden_dim = 1024                    # hidden layer size
+num_transformer_heads = 4           # number of attention heads
+positional_encoding = false         # enable/disable positional encoding
+dropout = 0.2                       # dropout between layers
+out_features = 256                  # output feature size before classification
+image_embedding_dim = 2560
+tr_layer_number = 2
+
+name_best_emo_model = "EmotionTransformer"
+hidden_dim_emo = 512
+out_features_emo = 1024
+tr_layer_number_emo = 2
+num_transformer_heads_emo = 16
+positional_encoding_emo = false
+path_to_saved_emotion_model = "results_emotiontransformer_2026-03-08_13-20-14/metrics_by_epoch/metrics_epochlog_EmotionTransformer_num_transformer_heads_16_20260308_145515_2026-03-08_14-55-15/best_model_dev.pt"
+
+name_best_per_model = "PersonalityTransformer"
+hidden_dim_per = 128
+out_features_per = 1024
+tr_layer_number_per = 2
+num_transformer_heads_per = 4
+positional_encoding_per = false
+path_to_saved_personality_model = "results_personalitytransformer_2026-03-11_01-20-41/metrics_by_epoch/metrics_epochlog_PersonalityTransformer_out_features_1024_20260311_014658_2026-03-11_01-46-58/best_model_dev.pt"
+
+
+
+
+# Loss weighting
+weight_emotion = 0.2
+weight_pers = 0.2
+ssl_weight_emotion = 1.0
+ssl_weight_personality = 1.0
+ssl_confidence_threshold_emo = 0.5
+ssl_confidence_threshold_pt = 0.5
+
+# Loss configuration
+pers_loss_type = "mae"              # options: ccc, mae, mse, rmse_bell, rmse_logcosh, RMGL
+emotion_loss_type = "BCE"            # classification loss for emotion
+flag_emo_weight = true             # use class weighting for emotion imbalance
+
+# GradNorm & SSL parameters
+alpha_sup = 1.0
+w_lr_sup = 0.005
+alpha_ssl = 1.5
+w_lr_ssl = 0.01
+lambda_ssl = 0.4
+w_floor = 1e-3
+
+
+# ─────────────────────────────
+# Optimizer Parameters
+# ─────────────────────────────
+[train.optimizer]
+optimizer = "adam"       # "adam", "adamw", "lion", "sgd", "rmsprop"
+lr = 1e-4                # learning rate
+weight_decay = 1e-5      # weight decay for regularization
+momentum = 0.9           # used only for SGD
+
+
+# ─────────────────────────────
+# Scheduler Parameters
+# ─────────────────────────────
+[train.scheduler]
+scheduler_type = "plateau"  # "none", "plateau", "cosine", "onecycle", or HuggingFace variants
+warmup_ratio = 0.1          # ratio of warmup iterations (0.1 = 10%)

inference.py

@@ -0,0 +1,151 @@
+# coding: utf-8
+import os, logging
+import random
+from pathlib import Path
+from typing import Dict, List
+import numpy as np
+import torch
+from utils.config_loader import ConfigLoader
+from torch.utils.data import ConcatDataset, DataLoader
+from tqdm import tqdm
+import whisper
+from models.models import MultiModalFusionModelWithAblation
+from modalities.vlm.feature_extractor import PretrainedVLMEmbeddingExtractor
+from modalities.text.feature_extractor import PretrainedTextEmbeddingExtractor
+
+def transform_matrix(matrix):
+    threshold1 = 1 - 1 / 7
+    threshold2 = 1 / 7
+    mask1 = matrix[:, 0] >= threshold1
+    result = np.zeros_like(matrix[:, 1:])
+    transformed = (matrix[:, 1:] >= threshold2).astype(int)
+    result[~mask1] = transformed[~mask1]
+    return result
+
+
+def process_predictions(pred_emo):
+    pred_emo = torch.nn.functional.softmax(pred_emo, dim=1).cpu().detach().numpy()
+    pred_emo = transform_matrix(pred_emo).tolist()
+    return pred_emo
+
+def aggregate(self, feats, average: bool = None):
+        """
+        Unified feature aggregation.
+
+        Args:
+            feats (Union[Tensor, dict, None]): Input features.
+            average (bool): If True — average over time (dim=1) when applicable.
+
+        Returns:
+            Aggregated features or None.
+
+        - If feats is a Tensor with shape [B, T, D] and average=True → average over T.
+        - If average=False → return as is.
+        - If feats is a dict → recurse over values.
+        - If feats is None → return None.
+        """
+
+        if average is None:
+            average = self.average_features
+
+        if feats is None:
+            return None
+
+        if isinstance(feats, torch.Tensor):
+            if average and feats.ndim == 3:
+                feats = feats.mean(dim=1)  # → [B, D]
+            return feats.squeeze()
+
+        if isinstance(feats, dict):
+            return {
+                key: self.aggregate(val, average)
+                for key, val in feats.items()
+            }
+
+        raise TypeError(f"Unsupported feature type: {type(feats)}")
+
+def transcribe_audio(audio_path):
+    whisper_model = whisper.load_model("base")
+    try:
+        result = whisper_model.transcribe(audio_path, fp16=False)
+        return result.get('text', '')
+    except Exception as e:
+        logging.error(f"Transcription failed: {e}")
+        return ""
+
+def predict(video_path):
+    base_config = ConfigLoader("config_copy.toml")
+    vlm_feature_extractor = PretrainedVLMEmbeddingExtractor(device=base_config.device)
+    text_feature_extractor = PretrainedTextEmbeddingExtractor(device=base_config.device)
+    modality_extractors = {
+        "video": vlm_feature_extractor,
+        "text": text_feature_extractor,
+    }
+    ablation_config = {}
+    if not base_config.single_task:
+        modality_combinations = [
+            [],  # 0 use all modalities
+
+            # Single modalities
+            ["text"],       # 1
+            ["video"]      # 2
+        ]
+
+        components = [
+            -1,
+            "disable_graph_attn",
+            "disable_cross_attn",
+            "disable_emo_logit_proj",
+            "disable_pkl_logit_proj",
+            "disable_guide_emo",
+            "disable_guide_pkl",
+        ]
+        ablation_config = (
+            {
+                "disabled_modalities": modality_combinations[base_config.id_ablation_type_by_modality],
+                components[base_config.id_ablation_type_by_component]: True
+            }
+            if components[base_config.id_ablation_type_by_component] != -1
+            else {"disabled_modalities": modality_combinations[base_config.id_ablation_type_by_modality]}
+        )
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    model = MultiModalFusionModelWithAblation(
+            hidden_dim=base_config.hidden_dim,
+            num_heads=base_config.num_transformer_heads,
+            dropout=base_config.dropout,
+            emo_out_dim=7,
+            pkl_out_dim=5,
+            device=device,
+            ablation_config=ablation_config,
+            attention=base_config.attention
+        ).to(device)
+    checkpoint = torch.load("multimodal_model.pt", map_location=device)
+    state_dict = checkpoint["model_state_dict"] if "model_state_dict" in checkpoint else checkpoint
+    model.load_state_dict(state_dict)
+    model.eval()
+    entry = {
+        "video_path": video_path,
+        "features": {},
+    }
+    try:
+        video_feats = modality_extractors["video"].extract(video_path=video_path, saved=False)
+        entry["features"]["video"] = aggregate(video_feats, True)
+    except Exception as e:
+        logging.warning(f"Video extract error {video_path}: {e}")
+        entry["features"]["video"] = None
+    try:
+        txt_raw = transcribe_audio(video_path)
+        text_feats = modality_extractors["text"].extract(txt_raw)
+        entry["features"]["text"] = aggregate(text_feats, True)
+    except Exception as e:
+        logging.warning(f"Text extract error {txt_raw}: {e}")
+        entry["features"]["text"] = None
+    outputs = model([entry])
+    
+    
+    if outputs.get('emotion_logits') is not None:
+        preds_emo = process_predictions(outputs['emotion_logits'])
+    if outputs.get('personality_scores') is not None:
+        preds_per = outputs['personality_scores']
+    return preds_emo, preds_per
+    

multimodal_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8a07a479cb927e51105a5d32989f8c7c38bdc9606f378fe34af6098d79d6fcb
+size 39054457

requirements.txt

@@ -0,0 +1,86 @@
+accelerate==1.12.0
+annotated-doc==0.0.4
+anyio==4.12.1
+av==16.1.0
+certifi==2026.2.25
+charset-normalizer==3.4.4
+click==8.3.1
+colorlog==6.10.1
+cuda-bindings==12.9.4
+cuda-pathfinder==1.4.0
+einops==0.8.2
+filelock==3.24.3
+fsspec==2026.2.0
+h11==0.16.0
+hf-xet==1.3.2
+httpcore==1.0.9
+httpx==0.28.1
+huggingface_hub==0.36.2
+idna==3.11
+Jinja2==3.1.6
+joblib==1.5.3
+markdown-it-py==4.0.0
+MarkupSafe==3.0.3
+mdurl==0.1.2
+mpmath==1.3.0
+networkx==3.6.1
+numpy==2.4.2
+nvidia-cublas==13.0.0.19
+nvidia-cublas-cu12==12.6.4.1
+nvidia-cuda-cupti==13.0.48
+nvidia-cuda-cupti-cu12==12.6.80
+nvidia-cuda-nvrtc==13.0.48
+nvidia-cuda-nvrtc-cu12==12.6.77
+nvidia-cuda-runtime==13.0.48
+nvidia-cuda-runtime-cu12==12.6.77
+nvidia-cudnn-cu12==9.10.2.21
+nvidia-cudnn-cu13==9.13.0.50
+nvidia-cufft==12.0.0.15
+nvidia-cufft-cu12==11.3.0.4
+nvidia-cufile==1.15.0.42
+nvidia-cufile-cu12==1.11.1.6
+nvidia-curand==10.4.0.35
+nvidia-curand-cu12==10.3.7.77
+nvidia-cusolver==12.0.3.29
+nvidia-cusolver-cu12==11.7.1.2
+nvidia-cusparse==12.6.2.49
+nvidia-cusparse-cu12==12.5.4.2
+nvidia-cusparselt-cu12==0.7.1
+nvidia-cusparselt-cu13==0.8.0
+nvidia-nccl-cu12==2.27.5
+nvidia-nccl-cu13==2.27.7
+nvidia-nvjitlink==13.0.39
+nvidia-nvjitlink-cu12==12.6.85
+nvidia-nvshmem-cu12==3.3.20
+nvidia-nvshmem-cu13==3.3.24
+nvidia-nvtx==13.0.39
+nvidia-nvtx-cu12==12.6.77
+packaging==26.0
+pandas==3.0.1
+pillow==12.1.1
+psutil==7.2.2
+Pygments==2.19.2
+python-dateutil==2.9.0.post0
+PyYAML==6.0.3
+qwen-vl-utils==0.0.14
+regex==2026.2.28
+requests==2.32.5
+rich==14.3.3
+safetensors==0.7.0
+scikit-learn==1.8.0
+scipy==1.17.1
+setuptools==82.0.0
+shellingham==1.5.4
+six==1.17.0
+sympy==1.14.0
+threadpoolctl==3.6.0
+tokenizers==0.22.2
+toml==0.10.2
+torchaudio==2.9.1+cu126
+torchvision==0.24.1+cu126
+tqdm==4.67.3
+triton==3.5.1
+typer==0.24.1
+typer-slim==0.24.0
+typing_extensions==4.15.0
+urllib3==2.6.3