Upload PerceiverForMaskedLM

config.json

@@ -6,21 +6,21 @@
   "attention_probs_dropout_prob": 0.1,
   "audio_samples_per_frame": 1920,
   "cross_attention_shape_for_attention": "kv",
-  "cross_attention_widening_factor": 4,
-  "d_latents": 20,
-  "d_model": 32,
+  "cross_attention_widening_factor": 1,
+  "d_latents": 16,
+  "d_model": 16,
   "hidden_act": "gelu",
   "image_size": 56,
   "initializer_range": 0.02,
   "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 7,
+  "max_position_embeddings": 2048,
   "model_type": "perceiver",
   "num_blocks": 1,
-  "num_cross_attention_heads": 1,
+  "num_cross_attention_heads": 2,
   "num_frames": 16,
   "num_latents": 10,
   "num_self_attends_per_block": 2,
-  "num_self_attention_heads": 1,
+  "num_self_attention_heads": 2,
   "output_shape": [
     1,
     16,
@@ -29,14 +29,14 @@
   ],
   "qk_channels": 256,
   "samples_per_patch": 16,
-  "self_attention_widening_factor": 4,
+  "self_attention_widening_factor": 1,
   "torch_dtype": "float32",
   "train_size": [
-    20,
-    20
+    368,
+    496
   ],
   "transformers_version": "4.22.0.dev0",
   "use_query_residual": true,
-  "v_channels": 1280,
-  "vocab_size": 99
+  "v_channels": 16,
+  "vocab_size": 262
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd5e9d6f47625a6cf5b9867b46deaae13ef0db29d8f78b05ba0b648c6df14d3a
-size 13896074
+oid sha256:384f9343dc0bebc2d4016dc7be6636d013f9d69292453ce2613168e70916c5d8
+size 461834