Training in progress, step 1000

config.json

@@ -0,0 +1,225 @@
+{
+  "architectures": [
+    "MSPVisualForCTC"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_msp_visual.MSPVisualConfig",
+    "AutoModelForCTC": "modeling_msp_visual.MSPVisualForCTC"
+  },
+  "bos_token_id": 1,
+  "ctc_loss_reduction": "mean",
+  "ctc_zero_infinity": true,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "final_dropout": 0.1,
+  "model_type": "msp_visual",
+  "pad_token_id": 0,
+  "transformers_version": "5.10.2",
+  "use_cache": false,
+  "visual_config": {
+    "_name_or_path": "MahmoodAnaam/avhubert_encoder_large_noise_pt_noise_ft_433h-V0",
+    "a_upsample_ratio": 1,
+    "activation_dropout": 0.0,
+    "activation_function": "relu",
+    "adapter_attn_dim": null,
+    "adapter_kernel_size": 3,
+    "adapter_stride": 2,
+    "add_adapter": false,
+    "adim": 1024,
+    "aheads": 12,
+    "apply_spec_augment": true,
+    "architectures": [
+      "AVHubertModel"
+    ],
+    "attention_dropout": 0.1,
+    "audio_dropout": 0.5,
+    "audio_feat_dim": 104,
+    "auto_map": {
+      "AutoConfig": "configuration_avhubert.AVHubertConfig",
+      "AutoModel": "modeling_avhubert.AVHubertModel"
+    },
+    "bos_token_id": 1,
+    "classifier_proj_size": 256,
+    "cnn_module_kernel": 31,
+    "codevector_dim": 256,
+    "contrastive_logits_temperature": 0.1,
+    "conv_bias": false,
+    "conv_channels": 1024,
+    "conv_dim": [
+      512,
+      512,
+      512,
+      512,
+      512,
+      512,
+      512
+    ],
+    "conv_kernel": [
+      10,
+      3,
+      3,
+      3,
+      3,
+      2,
+      2
+    ],
+    "conv_kernel_sizes": [
+      5,
+      5
+    ],
+    "conv_stride": [
+      5,
+      2,
+      2,
+      2,
+      2,
+      2,
+      2
+    ],
+    "ctc_loss_reduction": "sum",
+    "ctc_type": "builtin",
+    "ctc_zero_infinity": false,
+    "d_model": 1024,
+    "ddim": 1024,
+    "decoder_attention_heads": 8,
+    "decoder_ffn_dim": 4096,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 9,
+    "decoder_start_token_id": 2,
+    "dheads": 16,
+    "diversity_loss_weight": 0.1,
+    "dlayers": 6,
+    "do_stable_layer_norm": false,
+    "dropout": 0.1,
+    "dropout_features": 0.1,
+    "dropout_input": 0.1,
+    "dropout_rate": 0.1,
+    "dtype": "float32",
+    "dunits": 3072,
+    "elayers": 12,
+    "encoder_attention_heads": 16,
+    "encoder_embed_dim": 1024,
+    "encoder_ffn_dim": 2048,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 12,
+    "eos_token_id": 2,
+    "eunits": 3072,
+    "feat_extract_activation": "gelu",
+    "feat_extract_norm": "group",
+    "feat_proj_dropout": 0.1,
+    "feat_quantizer_dropout": 0.0,
+    "feature_grad_mult": 0.1,
+    "final_dim": 256,
+    "final_dropout": 0.0,
+    "freeze_feat_extract_train": true,
+    "fusion_hdim": 8192,
+    "fusion_norm": "batchnorm",
+    "hidden_act": "gelu",
+    "hidden_dropout": 0.1,
+    "hidden_size": 1024,
+    "init_std": 0.02,
+    "initializer_range": 0.02,
+    "input_channels": 1,
+    "input_feat_per_channel": 80,
+    "intermediate_size": 4096,
+    "label_rate": 25,
+    "layer_norm_eps": 1e-05,
+    "layerdrop": 0.0,
+    "logit_temp": 0.1,
+    "lsm_weight": 0.1,
+    "macaron_style": true,
+    "mask_channel_length": 10,
+    "mask_channel_min_space": 1,
+    "mask_channel_other": 0.0,
+    "mask_channel_prob": 0.0,
+    "mask_channel_selection": "static",
+    "mask_feature_length": 10,
+    "mask_feature_min_masks": 0,
+    "mask_feature_prob": 0.0,
+    "mask_length_audio": 10,
+    "mask_length_image": 5,
+    "mask_min_space": 1,
+    "mask_other": 0.0,
+    "mask_prob_audio": 0.8,
+    "mask_prob_image": 0.3,
+    "mask_selection": "static",
+    "mask_time_length": 10,
+    "mask_time_min_masks": 2,
+    "mask_time_min_space": 1,
+    "mask_time_other": 0.0,
+    "mask_time_prob": 0.0,
+    "mask_time_selection": "static",
+    "masking_type": "input",
+    "max_source_positions": 6000,
+    "max_target_positions": 2048,
+    "modality": "av",
+    "modality_dropout": 0.5,
+    "modality_fuse": "concat",
+    "model_type": "avhubert",
+    "mtlalpha": 0.1,
+    "no_mask_channel_overlap": false,
+    "no_mask_overlap": false,
+    "no_mask_time_overlap": false,
+    "num_adapter_layers": 3,
+    "num_attention_heads": 16,
+    "num_classes": 2004,
+    "num_codevector_groups": 2,
+    "num_codevectors_per_group": 320,
+    "num_conv_layers": 2,
+    "num_conv_pos_embedding_groups": 16,
+    "num_conv_pos_embeddings": 128,
+    "num_dictionaries": 1,
+    "num_feat_extract_layers": 7,
+    "num_hidden_layers": 24,
+    "num_negatives": 100,
+    "odim": 5049,
+    "output_hidden_size": 1024,
+    "pad_token_id": 1,
+    "proj_codevector_dim": 256,
+    "rel_pos_type": "latest",
+    "relu_type": "swish",
+    "resnet_relu_type": "prelu",
+    "resnet_weights": null,
+    "sample_rate": 25,
+    "scale_embedding": null,
+    "selection_type": "same_seq",
+    "sim_type": "cosine",
+    "skip_masked": false,
+    "skip_nomask": false,
+    "sub_encoder_layers": 0,
+    "target_glu": false,
+    "tdnn_dilation": [
+      1,
+      2,
+      3,
+      1,
+      1
+    ],
+    "tdnn_dim": [
+      512,
+      512,
+      512,
+      512,
+      1500
+    ],
+    "tdnn_kernel": [
+      5,
+      3,
+      3,
+      1,
+      1
+    ],
+    "transformer_attn_dropout_rate": 0.1,
+    "transformer_encoder_attn_layer_type": "rel_mha",
+    "transformer_input_layer": "conv3d",
+    "transformer_length_normalized_loss": false,
+    "untie_final_proj": true,
+    "use_cache": true,
+    "use_cnn_module": true,
+    "use_weighted_layer_sum": false,
+    "vocab_size": 1000,
+    "xvector_output_dim": 512,
+    "zero_triu": false
+  },
+  "vocab_size": 32
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31fb72fdc8486e5bc15d8553377fd6815a52b544e0553e7d15dbae5cea26a648
+size 1300783936

processing_msp_visual.py

@@ -0,0 +1,28 @@
+from transformers.processing_utils import ProcessorMixin
+
+
+class MSPVisualProcessor(ProcessorMixin):
+    attributes = ["video_processor", "tokenizer"]
+    video_processor_class = "MSPVisualVideoProcessor"
+    tokenizer_class = "AutoTokenizer"
+
+    def __init__(self, video_processor=None, tokenizer=None, **kwargs):
+        super().__init__(video_processor, tokenizer, **kwargs)
+        self.video_processor = video_processor
+        self.tokenizer = tokenizer
+
+    def __call__(self, videos=None, text=None, **kwargs):
+        if videos is None and text is None:
+            raise ValueError("Provide at least one of videos or text.")
+        inputs = super().__call__(
+            images=None, audio=None, videos=videos, text=text, **kwargs
+        )
+
+        if "input_ids" in inputs:
+            inputs["labels"] = inputs.pop("input_ids")
+
+        return inputs
+
+    @property
+    def model_input_names(self) -> list[str]:
+        return ["pixel_values_videos", "padding_mask_videos", "labels"]

processor_config.json

@@ -0,0 +1,31 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_msp_visual.MSPVisualProcessor"
+  },
+  "processor_class": "MSPVisualProcessor",
+  "video_processor": {
+    "auto_map": {
+      "AutoProcessor": "processing_msp_visual.MSPVisualProcessor",
+      "AutoVideoProcessor": "video_processing_msp_visual.MSPVisualVideoProcessor"
+    },
+    "crop_size": {
+      "height": 88,
+      "width": 88
+    },
+    "do_center_crop": true,
+    "do_convert_rgb_to_grayscale": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": 0.421,
+    "image_std": 0.165,
+    "resample": 2,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "height": 96,
+      "width": 96
+    },
+    "video_processor_type": "MSPVisualVideoProcessor"
+  }
+}

runs/Jul05_01-50-22_cdfd359d8cce/events.out.tfevents.1783216222.cdfd359d8cce.8290.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b5726ddea983bed10150dc93fc36058e5927583e69c9c355132d88b23b4f0ca
+size 17762

tokenizer_config.json

@@ -0,0 +1,64 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "4": {
+      "content": "|",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "auto_map": {
+    "AutoProcessor": "processing_msp_visual.MSPVisualProcessor"
+  },
+  "backend": "custom",
+  "bos_token": "<s>",
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "word_delimiter_token": "|"
+  },
+  "pad_token": "<pad>",
+  "processor_class": "MSPVisualProcessor",
+  "replace_word_delimiter_char": " ",
+  "target_lang": null,
+  "tokenizer_class": "Wav2Vec2CTCTokenizer",
+  "unk_token": "<unk>",
+  "word_delimiter_token": "|"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea7fce6a33cd52626ca2d7dd4a36749ff97abca33a7216c82c5b5e39a08f3350
+size 5265

video_processing_msp_visual.py

@@ -0,0 +1,134 @@
+import torch
+import torchvision.transforms.v2.functional as tvF
+from torchcodec.decoders import VideoDecoder
+from transformers.image_processing_utils import BatchFeature
+from transformers.image_utils import PILImageResampling
+from transformers.processing_utils import Unpack, VideosKwargs
+from transformers.video_processing_utils import BaseVideoProcessor, VideoMetadata
+from transformers.video_utils import VideoInput
+
+
+class MSPVisualVideoProcessor(BaseVideoProcessor):
+    resample = PILImageResampling.BILINEAR
+
+    def __init__(
+        self,
+        do_convert_rgb_to_grayscale: bool = True,
+        do_rescale: bool = True,
+        rescale_factor: float = 1 / 255.0,
+        image_mean=0.421,
+        image_std=0.165,
+        do_normalize: bool = True,
+        do_resize: bool = True,
+        size: dict[str, int] = {"height": 96, "width": 96},
+        do_center_crop: bool = True,
+        crop_size: dict[str, int] = {"height": 88, "width": 88},
+        **kwargs: Unpack[VideosKwargs],
+    ):
+        super().__init__(
+            do_rescale=do_rescale,
+            rescale_factor=rescale_factor,
+            image_mean=image_mean,
+            image_std=image_std,
+            do_normalize=do_normalize,
+            do_resize=do_resize,
+            size=size,
+            do_center_crop=do_center_crop,
+            crop_size=crop_size,
+            **kwargs,
+        )
+        self.do_convert_rgb_to_grayscale = do_convert_rgb_to_grayscale
+
+    def sample_frames(
+        self,
+        metadata: VideoMetadata,
+        num_frames: int | None = None,
+        fps: int | float | None = None,
+        **kwargs,
+    ):
+        if num_frames:
+            total_frames = metadata.total_num_frames
+            num_frames = num_frames if num_frames is not None else self.num_frames
+            assert num_frames is not None, (
+                "`num_frames` must be specified if `fixed_len_video == True`"
+            )
+            frame_idxs = [
+                int(i * (total_frames - 1) / (num_frames - 1))
+                for i in range(num_frames)
+            ]
+            return torch.tensor(frame_idxs)
+        else:
+            return super().sample_frames(metadata, num_frames, fps, **kwargs)
+
+    def _load_video(self, src: str | bytes) -> torch.Tensor:
+        """
+        Load video from a file path or bytes and return as a 4D torch.Tensor.
+        Args:
+            src (str | bytes): Path to the video file or bytes of the video file.
+        Returns:
+            torch.Tensor: Loaded video as a 4D tensor (num_frames, height, width, num_channels).
+        """
+        vd = VideoDecoder(src)
+        video = vd.get_frames_in_range(0, vd.metadata.num_frames).data
+        return video
+
+    def __call__(
+        self, videos: VideoInput | str | list[str] | bytes | list[bytes], **kwargs
+    ):
+        """Overrides the __call__ method to handle video input as file paths or bytes."""
+        if isinstance(videos, (str, bytes)):
+            videos = self._load_video(videos)
+        elif isinstance(videos, list) and isinstance(videos[0], (str, bytes)):
+            videos = [self._load_video(v) for v in videos]
+
+        # remove kwargs not in VideosKwargs
+        # for key in list(kwargs.keys()):
+        #     if key not in VideosKwargs.__optional_keys__:
+        #         kwargs.pop(key, None)
+        return super().__call__(videos, **kwargs)
+
+    def convert_rgb_to_grayscale(self, video: torch.Tensor) -> torch.Tensor:
+        """
+        Convert a video to grayscale.
+        """
+        video = tvF.rgb_to_grayscale(video)
+        return video
+
+    def _preprocess(
+        self,
+        videos: VideoInput,
+        **kwargs: Unpack[VideosKwargs],
+    ) -> BatchFeature:
+        """
+        Preprocesses a video or a batch of videos.
+        Args:
+            videos (VideoInput): Video to preprocess.
+                See `VideoInput` for details.
+            **kwargs: Additional keyword arguments.
+        Returns:
+            BatchFeature: A BatchFeature with the following fields:
+                - pixel_values_videos: Pixel values to be fed to a model, of shape (batch_size,num_channels, num_frames, height, width).
+                - padding_mask_videos (optional): Mask to be used for padding, of shape (batch_size, num_frames).
+        """
+
+        # Always set `return_tensors` to `None` since it won't pad variable length videos
+        # We'll handle this after we call the parent' method
+        return_tensors = kwargs.pop("return_tensors", None)
+        result = super()._preprocess(videos, **kwargs)
+        pixels = result.pixel_values_videos
+        if self.do_convert_rgb_to_grayscale:
+            pixels = [self.convert_rgb_to_grayscale(video) for video in pixels]
+        data = {"pixel_values_videos": pixels}
+        if return_tensors:
+            lengths = torch.tensor([video.size(0) for video in pixels])
+            pixels = torch.nn.utils.rnn.pad_sequence(
+                pixels, batch_first=True, padding_value=0.0
+            )
+            data["pixel_values_videos"] = pixels
+            if lengths.unique().size(0) > 1:
+                mask = torch.arange(lengths.max())[None] < lengths[:, None]
+                data["padding_mask_videos"] = mask
+        # pixel_values_videos shape [batch_size, num_channels, num_frames, height, width]
+        data["pixel_values_videos"] = data["pixel_values_videos"].permute(0, 2, 1, 3, 4)
+
+        return BatchFeature(data=data, tensor_type=return_tensors)

vocab.json

@@ -0,0 +1,34 @@
+{
+  "'": 26,
+  "</s>": 2,
+  "<pad>": 0,
+  "<s>": 1,
+  "<unk>": 3,
+  "A": 8,
+  "B": 24,
+  "C": 20,
+  "D": 15,
+  "E": 5,
+  "F": 22,
+  "G": 21,
+  "H": 11,
+  "I": 9,
+  "J": 28,
+  "K": 25,
+  "L": 14,
+  "M": 19,
+  "N": 10,
+  "O": 7,
+  "P": 23,
+  "Q": 30,
+  "R": 13,
+  "S": 12,
+  "T": 6,
+  "U": 16,
+  "V": 27,
+  "W": 18,
+  "X": 29,
+  "Y": 17,
+  "Z": 31,
+  "|": 4
+}