compat: vendor _expand_mask/clip_loss, fix tokenizer init, alias functional_tensor, guard set_audio_backend

languagebind/audio/modeling_audio.py

@@ -9,7 +9,24 @@ from torch.nn import functional as F
 from transformers import PreTrainedModel, add_start_docstrings
 from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
 from transformers.models.clip.modeling_clip import CLIPMLP, CLIPAttention, CLIPTextEmbeddings, CLIPVisionEmbeddings, \
-    CLIPVisionModelWithProjection, CLIPTextModelWithProjection, _expand_mask, CLIPOutput, clip_loss
+    CLIPVisionModelWithProjection, CLIPTextModelWithProjection, CLIPOutput
+
+
+def _expand_mask(mask, dtype, tgt_len=None):
+    # Vendored from transformers<4.35; removed upstream in the attention-mask refactor.
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
+    inverted_mask = 1.0 - expanded_mask
+    return inverted_mask.masked_fill(inverted_mask.to(bool), float('-inf'))
+
+
+def clip_loss(similarity):
+    # Vendored from transformers; removed upstream.
+    import torch
+    caption_loss = torch.nn.functional.cross_entropy(similarity, torch.arange(len(similarity), device=similarity.device))
+    image_loss = torch.nn.functional.cross_entropy(similarity.t(), torch.arange(len(similarity), device=similarity.device))
+    return (caption_loss + image_loss) / 2.0
 from transformers.utils import add_start_docstrings_to_model_forward, replace_return_docstrings
 
 from .configuration_audio import LanguageBindAudioConfig, CLIPVisionConfig, CLIPTextConfig

languagebind/audio/processing_audio.py

@@ -14,7 +14,10 @@ def make_list_of_images(x):
     return x
 
 
-torchaudio.set_audio_backend("soundfile")
+# torchaudio.set_audio_backend was deprecated and later removed; it is a no-op
+# on modern torchaudio, so only call it when present (older versions).
+if hasattr(torchaudio, "set_audio_backend"):
+    torchaudio.set_audio_backend("soundfile")
 
 def torchaudio_loader(path):
     return torchaudio.load(path)

languagebind/audio/tokenization_audio.py

@@ -67,11 +67,11 @@ class LanguageBindAudioTokenizer(CLIPTokenizer):
             **kwargs,
     ):
         super(LanguageBindAudioTokenizer, self).__init__(
-            vocab_file,
-            merges_file,
-            errors,
-            unk_token,
-            bos_token,
-            eos_token,
-            pad_token,  # hack to enable padding
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,  # hack to enable padding
             **kwargs,)

languagebind/depth/modeling_depth.py

@@ -9,7 +9,24 @@ from torch.nn import functional as F
 from transformers import PreTrainedModel, add_start_docstrings
 from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
 from transformers.models.clip.modeling_clip import CLIPMLP, CLIPAttention, CLIPTextEmbeddings, CLIPVisionEmbeddings, \
-    CLIPVisionModelWithProjection, CLIPTextModelWithProjection, _expand_mask, CLIPOutput, clip_loss
+    CLIPVisionModelWithProjection, CLIPTextModelWithProjection, CLIPOutput
+
+
+def _expand_mask(mask, dtype, tgt_len=None):
+    # Vendored from transformers<4.35; removed upstream in the attention-mask refactor.
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
+    inverted_mask = 1.0 - expanded_mask
+    return inverted_mask.masked_fill(inverted_mask.to(bool), float('-inf'))
+
+
+def clip_loss(similarity):
+    # Vendored from transformers; removed upstream.
+    import torch
+    caption_loss = torch.nn.functional.cross_entropy(similarity, torch.arange(len(similarity), device=similarity.device))
+    image_loss = torch.nn.functional.cross_entropy(similarity.t(), torch.arange(len(similarity), device=similarity.device))
+    return (caption_loss + image_loss) / 2.0
 from transformers.utils import add_start_docstrings_to_model_forward, replace_return_docstrings
 
 from .configuration_depth import LanguageBindDepthConfig, CLIPVisionConfig, CLIPTextConfig

languagebind/depth/tokenization_depth.py

@@ -67,11 +67,11 @@ class LanguageBindDepthTokenizer(CLIPTokenizer):
             **kwargs,
     ):
         super(LanguageBindDepthTokenizer, self).__init__(
-            vocab_file,
-            merges_file,
-            errors,
-            unk_token,
-            bos_token,
-            eos_token,
-            pad_token,  # hack to enable padding
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,  # hack to enable padding
             **kwargs,)

languagebind/image/modeling_image.py

@@ -9,7 +9,24 @@ from torch.nn import functional as F
 from transformers import PreTrainedModel, add_start_docstrings
 from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
 from transformers.models.clip.modeling_clip import CLIPMLP, CLIPAttention, CLIPTextEmbeddings, CLIPVisionEmbeddings, \
-    CLIPVisionModelWithProjection, CLIPTextModelWithProjection, _expand_mask, CLIPOutput, clip_loss
+    CLIPVisionModelWithProjection, CLIPTextModelWithProjection, CLIPOutput
+
+
+def _expand_mask(mask, dtype, tgt_len=None):
+    # Vendored from transformers<4.35; removed upstream in the attention-mask refactor.
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
+    inverted_mask = 1.0 - expanded_mask
+    return inverted_mask.masked_fill(inverted_mask.to(bool), float('-inf'))
+
+
+def clip_loss(similarity):
+    # Vendored from transformers; removed upstream.
+    import torch
+    caption_loss = torch.nn.functional.cross_entropy(similarity, torch.arange(len(similarity), device=similarity.device))
+    image_loss = torch.nn.functional.cross_entropy(similarity.t(), torch.arange(len(similarity), device=similarity.device))
+    return (caption_loss + image_loss) / 2.0
 from transformers.utils import add_start_docstrings_to_model_forward, replace_return_docstrings
 
 from .configuration_image import LanguageBindImageConfig, CLIPVisionConfig, CLIPTextConfig

languagebind/image/tokenization_image.py

@@ -67,11 +67,11 @@ class LanguageBindImageTokenizer(CLIPTokenizer):
             **kwargs,
     ):
         super(LanguageBindImageTokenizer, self).__init__(
-            vocab_file,
-            merges_file,
-            errors,
-            unk_token,
-            bos_token,
-            eos_token,
-            pad_token,  # hack to enable padding
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,  # hack to enable padding
             **kwargs,)

languagebind/thermal/modeling_thermal.py

@@ -9,7 +9,24 @@ from torch.nn import functional as F
 from transformers import PreTrainedModel, add_start_docstrings
 from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
 from transformers.models.clip.modeling_clip import CLIPMLP, CLIPAttention, CLIPTextEmbeddings, CLIPVisionEmbeddings, \
-    CLIPVisionModelWithProjection, CLIPTextModelWithProjection, _expand_mask, CLIPOutput, clip_loss
+    CLIPVisionModelWithProjection, CLIPTextModelWithProjection, CLIPOutput
+
+
+def _expand_mask(mask, dtype, tgt_len=None):
+    # Vendored from transformers<4.35; removed upstream in the attention-mask refactor.
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
+    inverted_mask = 1.0 - expanded_mask
+    return inverted_mask.masked_fill(inverted_mask.to(bool), float('-inf'))
+
+
+def clip_loss(similarity):
+    # Vendored from transformers; removed upstream.
+    import torch
+    caption_loss = torch.nn.functional.cross_entropy(similarity, torch.arange(len(similarity), device=similarity.device))
+    image_loss = torch.nn.functional.cross_entropy(similarity.t(), torch.arange(len(similarity), device=similarity.device))
+    return (caption_loss + image_loss) / 2.0
 from transformers.utils import add_start_docstrings_to_model_forward, replace_return_docstrings
 
 from .configuration_thermal import LanguageBindThermalConfig, CLIPVisionConfig, CLIPTextConfig

languagebind/thermal/tokenization_thermal.py

@@ -67,11 +67,11 @@ class LanguageBindThermalTokenizer(CLIPTokenizer):
             **kwargs,
     ):
         super(LanguageBindThermalTokenizer, self).__init__(
-            vocab_file,
-            merges_file,
-            errors,
-            unk_token,
-            bos_token,
-            eos_token,
-            pad_token,  # hack to enable padding
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,  # hack to enable padding
             **kwargs,)

languagebind/video/modeling_video.py

@@ -9,7 +9,24 @@ from torch.nn import functional as F
 from transformers import PreTrainedModel, add_start_docstrings
 from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
 from transformers.models.clip.modeling_clip import CLIPMLP, CLIPAttention, CLIPTextEmbeddings, CLIPVisionEmbeddings, \
-    CLIPVisionModelWithProjection, CLIPTextModelWithProjection, _expand_mask, CLIPOutput, clip_loss
+    CLIPVisionModelWithProjection, CLIPTextModelWithProjection, CLIPOutput
+
+
+def _expand_mask(mask, dtype, tgt_len=None):
+    # Vendored from transformers<4.35; removed upstream in the attention-mask refactor.
+    bsz, src_len = mask.size()
+    tgt_len = tgt_len if tgt_len is not None else src_len
+    expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
+    inverted_mask = 1.0 - expanded_mask
+    return inverted_mask.masked_fill(inverted_mask.to(bool), float('-inf'))
+
+
+def clip_loss(similarity):
+    # Vendored from transformers; removed upstream.
+    import torch
+    caption_loss = torch.nn.functional.cross_entropy(similarity, torch.arange(len(similarity), device=similarity.device))
+    image_loss = torch.nn.functional.cross_entropy(similarity.t(), torch.arange(len(similarity), device=similarity.device))
+    return (caption_loss + image_loss) / 2.0
 from transformers.utils import add_start_docstrings_to_model_forward, replace_return_docstrings
 
 from .configuration_video import LanguageBindVideoConfig, CLIPVisionConfig, CLIPTextConfig

languagebind/video/processing_video.py

@@ -10,6 +10,17 @@ from transformers.image_processing_utils import BatchFeature
 from pytorchvideo.data.encoded_video import EncodedVideo
 from torchvision.transforms import Compose, Lambda, ToTensor
 from torchvision.transforms._transforms_video import NormalizeVideo, RandomCropVideo, RandomHorizontalFlipVideo, CenterCropVideo
+# pytorchvideo 0.1.5 imports torchvision.transforms.functional_tensor at module
+# import time, but torchvision removed that module in newer releases. LanguageBind
+# only needs ApplyTransformToKey, ShortSideScale, and UniformTemporalSubsample from
+# pytorchvideo.transforms, so provide the old module name before importing it.
+import sys
+import torchvision.transforms.functional as _torchvision_functional
+
+sys.modules.setdefault(
+    "torchvision.transforms.functional_tensor",
+    _torchvision_functional,
+)
 from pytorchvideo.transforms import ApplyTransformToKey, ShortSideScale, UniformTemporalSubsample
 
 decord.bridge.set_bridge('torch')

languagebind/video/tokenization_video.py

@@ -67,11 +67,11 @@ class LanguageBindVideoTokenizer(CLIPTokenizer):
             **kwargs,
     ):
         super(LanguageBindVideoTokenizer, self).__init__(
-            vocab_file,
-            merges_file,
-            errors,
-            unk_token,
-            bos_token,
-            eos_token,
-            pad_token,  # hack to enable padding
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,  # hack to enable padding
             **kwargs,)