data

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<fake_token_around_image>": 50265,
+  "<image>": 50266
+}

config.json

@@ -0,0 +1,48 @@
+{
+  "_remove_final_layer_norm": false,
+  "activation_function": "relu",
+  "additional_vocab_size": 2,
+  "alpha_initializer": "ones",
+  "alpha_type": "vector",
+  "alphas_initializer_range": 0.0,
+  "architectures": [
+    "VOPTForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 2,
+  "cross_layer_interval": 1,
+  "do_layer_norm_before": true,
+  "dropout": 0.1,
+  "eos_token_id": 2,
+  "ffn_dim": 64,
+  "freeze_lm_head": false,
+  "freeze_text_layers": true,
+  "freeze_text_module_exceptions": [],
+  "freeze_vision_layers": true,
+  "freeze_vision_module_exceptions": [],
+  "hidden_size": 16,
+  "image_token_index": 50257,
+  "init_std": 0.02,
+  "layerdrop": 0.0,
+  "max_new_tokens": 100,
+  "max_position_embeddings": 128,
+  "model_type": "opt",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "pad_token_id": 1,
+  "resampler_depth": 2,
+  "resampler_head_dim": 8,
+  "resampler_n_heads": 2,
+  "resampler_n_latents": 16,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.26.0.dev0",
+  "use_cache": true,
+  "use_resampler": true,
+  "vision_embed_dim": 32,
+  "vision_image_size": 30,
+  "vision_model_name": "hf-internal-testing/tiny-random-clip",
+  "vision_model_params": "{}",
+  "vocab_size": 50265,
+  "word_embed_proj_dim": 16
+}

make_tiny_model.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+# This script creates a super tiny model that is useful inside tests, when we just want to test that
+# the machinery works, without needing to check the quality of the outcomes.
+#
+# usage: adjust the configs if wanted, but otherwise just run the script
+
+from pathlib import Path
+from types import SimpleNamespace
+
+import torchvision.transforms as transforms
+from PIL import Image
+
+from m4.models.vopt.modeling_vopt import VOPTConfig, VOPTForCausalLM
+from m4.training.packing import image_attention_mask_for_packed_input_ids, incremental_to_binary_attention_mask
+from m4.training.utils import get_tokenizer
+
+
+mname_tiny = "tiny-random-vopt-clip"
+
+path = Path(mname_tiny)
+path.mkdir(parents=True, exist_ok=True)
+
+# from the hardcoded https://github.com/huggingface/m4/blob/adf102f0000cb2632cd8a3ebb87398c65e448a97/m4/training/main.py#L80
+additional_vocab_size = 2
+
+config = VOPTConfig()
+config.update(
+    dict(
+        ffn_dim=64,
+        hidden_size=16,
+        max_position_embeddings=128,
+        num_attention_heads=4,
+        num_hidden_layers=2,
+        word_embed_proj_dim=16,
+        max_new_tokens=100,
+        use_resampler=True,
+        resampler_depth=2,
+        resampler_head_dim=8,
+        resampler_n_heads=2,
+        resampler_n_latents=16,
+        vision_embed_dim=32,
+        vision_image_size=30,
+        vision_model_name="hf-internal-testing/tiny-random-clip",
+        vision_model_params="{}",
+        vocab_size=50265,
+        additional_vocab_size=additional_vocab_size,
+    )
+)
+
+# print(config)
+# can now modify config to say tiny values
+
+model = VOPTForCausalLM.from_config(config)
+# print(model.config)
+# print(model)
+
+tokenizer_config = dict(
+    tokenizer_add_special_tokens="{}",
+    tokenizer_add_tokens=(
+        '[AddedToken("<fake_token_around_image>", rstrip=False, lstrip=False), AddedToken("<image>", rstrip=False,'
+        " lstrip=False)]"
+    ),
+    tokenizer_name="facebook/opt-13b",
+    tokenizer_params='{"use_fast":True}',
+)
+tokenizer_config = SimpleNamespace(**tokenizer_config)
+# print(tokenizer_config)
+
+tokenizer = get_tokenizer(
+    tokenizer_name=tokenizer_config.tokenizer_name,
+    tokenizer_add_tokens=tokenizer_config.tokenizer_add_tokens,
+    tokenizer_add_special_tokens=tokenizer_config.tokenizer_add_special_tokens,
+    tokenizer_params=tokenizer_config.tokenizer_params,
+    additional_vocab_size=model.config.additional_vocab_size,
+    model_vocab_size=model.config.vocab_size,
+)
+assert "<image>" in tokenizer.get_vocab()
+
+# Test w/ one image and one text
+query = "<fake_token_around_image><image><fake_token_around_image>This is a picture of a cat."
+query_tokens = tokenizer(query, return_tensors="pt")
+
+num_images_per_ex = 1
+pixel_values = transforms.ToTensor()(Image.new("RGB", (30, 30))).repeat(1, 1, 1, 1).unsqueeze(0)
+image_attention_mask, _ = image_attention_mask_for_packed_input_ids(query_tokens["input_ids"], tokenizer)
+image_attention_mask = incremental_to_binary_attention_mask(image_attention_mask, num_classes=num_images_per_ex)
+
+input = {
+    "input_ids": query_tokens["input_ids"],
+    "attention_mask": query_tokens["attention_mask"],
+    "pixel_values": pixel_values,
+    "pixel_values": pixel_values,
+    "image_attention_mask": image_attention_mask,
+}
+# debug shapes
+# print(query_tokens["input_ids"].shape)
+# print(query_tokens["attention_mask"].shape)
+# print(pixel_values.shape)
+# print(image_attention_mask.shape)
+
+out_gen = model.generate(**input)
+text = tokenizer.batch_decode(out_gen)
+# print(text)
+
+# Save model + config + tokenizer
+model.half()  # makes it smaller
+model.save_pretrained(path)
+tokenizer.save_pretrained(path)
+
+# test we can load it back
+model = VOPTForCausalLM.from_pretrained(path)
+
+print(f"Generated {mname_tiny} - Upload the generated folder to the hub")

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b026c0c74c4616842ce379de3a50d64c83a6f4a261d9d333e4f115764951a48e
+size 3435021

special_tokens_map.json

@@ -0,0 +1,46 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,41 @@
+{
+  "add_bos_token": true,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1000000000000000019884624838656,
+  "name_or_path": "facebook/opt-13b",
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "special_tokens_map_file": null,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}