added files

README.md

@@ -0,0 +1,50 @@
+---
+license: apache-2.0
+---
+
+# OFA-tiny
+
+## Introduction
+This is the **tiny** version of OFA pretrained model finetuned on CLEVR and a custom block stack dataset. 
+
+The directory includes 4 files, namely `config.json` which consists of model configuration, `vocab.json` and `merge.txt` for our OFA tokenizer, and lastly `pytorch_model.bin` which consists of model weights.
+
+
+## How to use
+Download the models as shown below.
+```bash
+git clone https://github.com/sohananisetty/OFA_VQA.git
+git clone https://huggingface.co/SohanAnisetty/ofa-vqa-base
+```
+
+After, refer the path to ofa-vqa-base to `ckpt_dir`, and prepare an image for the testing example below. 
+
+```python
+from PIL import Image
+from torchvision import transforms
+from transformers import OFATokenizer, OFAModelForVQA
+
+mean, std = [0.5, 0.5, 0.5], [0.5, 0.5, 0.5]
+resolution = 480
+patch_resize_transform = transforms.Compose([
+        lambda image: image.convert("RGB"),
+        transforms.Resize((resolution, resolution), interpolation=Image.BICUBIC),
+        transforms.ToTensor(), 
+        transforms.Normalize(mean=mean, std=std)
+    ])
+
+
+tokenizer = OFATokenizer.from_pretrained(ckpt_dir)
+
+txt = " what does the image describe?"
+inputs = tokenizer([txt], return_tensors="pt").input_ids
+inputs = inputs.cuda()
+img = Image.open(path_to_image)
+patch_img = patch_resize_transform(img).unsqueeze(0).cuda()
+
+
+model = OFAModel.from_pretrained(ckpt_dir, use_cache=False).cuda()
+gen = model.generate(inputs, patch_images=patch_img, num_beams=5, no_repeat_ngram_size=3) 
+
+print(tokenizer.batch_decode(gen skip_special_tokens=True))
+```

config.json

@@ -0,0 +1,52 @@
+{
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "add_type_embedding": true,
+  "architectures": [
+    "OFAModelForVQA"
+  ],
+  "attention_dropout": 0.0,
+  "attn_scale_factor": 2.0,
+  "bos_token_id": 0,
+  "classifier_dropout": 0.0,
+  "code_image_size": 128,
+  "code_layernorm_embedding": true,
+  "d_model": 768,
+  "decoder_attention_heads": 12,
+  "decoder_drop_path_rate": 0.0,
+  "decoder_ffn_dim": 3072,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 6,
+  "decoder_normalize_before": true,
+  "decoder_start_token_id": 0,
+  "dropout": 0.1,
+  "encoder_attention_heads": 12,
+  "encoder_drop_path_rate": 0.0,
+  "encoder_ffn_dim": 3072,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 6,
+  "encoder_normalize_before": true,
+  "entangle_position_embedding": false,
+  "eos_token_id": 2,
+  "forced_eos_token_id": 2,
+  "image_bucket_size": 42,
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "layernorm_embedding": true,
+  "max_position_embeddings": 1024,
+  "model_type": "ofa",
+  "normformer": true,
+  "num_hidden_layers": 6,
+  "pad_token_id": 1,
+  "patch_layernorm_embedding": true,
+  "resnet_drop_path_rate": 0.0,
+  "resnet_model_path": null,
+  "resnet_type": "resnet101",
+  "scale_embedding": false,
+  "share_decoder_input_output_embed": true,
+  "token_bucket_size": 256,
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.1",
+  "use_cache": false,
+  "vocab_size": 59457
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 2,
+  "forced_eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.26.1",
+  "use_cache": false
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3b36e31e3670185941cd270e0d7022ed80ab00f52fe9d4b245966e82e603353
+size 796223833