Add exported weights

anyup.py

@@ -76,53 +76,6 @@ class ResBlock(nn.Module):
 # LearnedFeatureUnification (from layers/feature_unification.py)
 # ---------------------------------------------------------------------------
 
-compute_basis_size = {
-    "gauss_deriv": lambda order, mirror: ((order + 1) * (order + 2))
-    // (1 if mirror else 2)
-}
-
-
-def herme_vander_torch(z, m):
-    He0 = z.new_ones(z.shape)
-    if m == 0:
-        return He0[:, None]
-    H = [He0, z]
-    for n in range(1, m):
-        H.append(z * H[-1] - n * H[-2])
-    return torch.stack(H, 1)
-
-
-def gauss_deriv(
-    max_order,
-    device,
-    dtype,
-    kernel_size,
-    sigma=None,
-    include_negations=False,
-    scale_magnitude=True,
-):
-    sigma = (kernel_size // 2) / 1.645 if sigma is None else sigma
-    if kernel_size % 2 == 0:
-        raise ValueError("ksize must be odd")
-    half = kernel_size // 2
-    x = torch.arange(-half, half + 1, dtype=dtype, device=device)
-    z = x / sigma
-    g = torch.exp(-0.5 * z**2) / (sigma * (2.0 * torch.pi) ** 0.5)
-    He = herme_vander_torch(z, max_order)
-    derivs_1d = [
-        (((-1) ** n) / (sigma**n) if scale_magnitude else (-1) ** n) * He[:, n] * g
-        for n in range(max_order + 1)
-    ]
-    bank = []
-    for o in range(max_order + 1):
-        for i in range(o + 1):
-            K = torch.outer(derivs_1d[o - i], derivs_1d[i])
-            bank.append(K)
-            if include_negations:
-                bank.append(-K)
-    return torch.stack(bank, 0)
-
-
 class LearnedFeatureUnification(nn.Module):
     def __init__(
         self,
@@ -133,26 +86,9 @@ class LearnedFeatureUnification(nn.Module):
         super().__init__()
         self.out_channels = out_channels
         self.kernel_size = kernel_size
-        if init_gaussian_derivatives:
-            order = 0
-            while compute_basis_size["gauss_deriv"](order, False) < out_channels:
-                order += 1
-            print(
-                f"FeatureUnification: initializing with Gaussian derivative basis of order {order}"
-            )
-            self.basis = nn.Parameter(
-                gauss_deriv(
-                    order,
-                    device="cpu",
-                    dtype=torch.float32,
-                    kernel_size=kernel_size,
-                    scale_magnitude=False,
-                )[:out_channels, None]
-            )
-        else:
-            self.basis = nn.Parameter(
-                torch.randn(out_channels, 1, kernel_size, kernel_size)
-            )
+        self.basis = nn.Parameter(
+            torch.randn(out_channels, 1, kernel_size, kernel_size)
+        )
 
     def forward(self, features: torch.Tensor) -> torch.Tensor:
         b, c, h, w = features.shape

config.json

@@ -7,8 +7,7 @@
     "AutoModelForCausalLM": "modeling_falcon_perception.FalconPerceptionForSegmentation"
   },
   "model_type": "falcon_perception",
-  "torch_dtype": "bfloat16",
-
+  "torch_dtype": "float32",
   "dim": 1024,
   "n_layers": 28,
   "n_heads": 16,
@@ -19,38 +18,27 @@
   "norm_eps": 1e-05,
   "max_seq_len": 8192,
   "rope_theta": 10000,
-
   "channel_size": 3,
   "spatial_patch_size": 16,
   "temporal_patch_size": 1,
-
   "do_segmentation": true,
   "segm_out_dim": 256,
   "num_segm_layers": 3,
-
   "coord_enc_dim": 512,
   "coord_dec_dim": 8192,
   "coord_out_dim": 2048,
   "coord_token_id": 240,
-
   "size_enc_dim": 512,
   "size_dec_dim": 8192,
   "size_out_dim": 2048,
   "size_token_id": 241,
-
   "seg_token_id": 262,
-  "img_id": 227,
   "eos_id": 11,
+  "img_id": 227,
   "image_cls_token_id": 244,
-  "image_mask_token_id": 243,
   "image_reg_1_token_id": 245,
   "image_reg_2_token_id": 246,
   "image_reg_3_token_id": 247,
   "image_reg_4_token_id": 248,
-  "img_start_id": 229,
-  "img_end_id": 230,
-  "img_row_sep_id": 228,
-  "vid_start_id": 231,
-  "vid_end_id": 232,
-  "frame_sep_id": 233
-}
+  "img_end_id": 230
+}

configuration_falcon_perception.py

@@ -31,20 +31,14 @@ class FalconPerceptionConfig(PretrainedConfig):
         size_out_dim: int = 2048,
         size_token_id: int = 241,
         seg_token_id: int = 262,
-        img_id: int = 227,
         eos_id: int = 11,
+        img_id: int = 227,
         image_cls_token_id: int = 244,
-        image_mask_token_id: int = 243,
         image_reg_1_token_id: int = 245,
         image_reg_2_token_id: int = 246,
         image_reg_3_token_id: int = 247,
         image_reg_4_token_id: int = 248,
-        img_start_id: int = 229,
         img_end_id: int = 230,
-        img_row_sep_id: int = 228,
-        vid_start_id: int = 231,
-        vid_end_id: int = 232,
-        frame_sep_id: int = 233,
         **kwargs,
     ):
         self.dim = dim
@@ -72,18 +66,12 @@ class FalconPerceptionConfig(PretrainedConfig):
         self.size_out_dim = size_out_dim
         self.size_token_id = size_token_id
         self.seg_token_id = seg_token_id
-        self.img_id = img_id
         self.eos_id = eos_id
+        self.img_id = img_id
         self.image_cls_token_id = image_cls_token_id
-        self.image_mask_token_id = image_mask_token_id
         self.image_reg_1_token_id = image_reg_1_token_id
         self.image_reg_2_token_id = image_reg_2_token_id
         self.image_reg_3_token_id = image_reg_3_token_id
         self.image_reg_4_token_id = image_reg_4_token_id
-        self.img_start_id = img_start_id
         self.img_end_id = img_end_id
-        self.img_row_sep_id = img_row_sep_id
-        self.vid_start_id = vid_start_id
-        self.vid_end_id = vid_end_id
-        self.frame_sep_id = frame_sep_id
         super().__init__(**kwargs)

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17b7e908aee8afc1ade0af64e5107d2dc63bb9a287acc407041c3ef5e4278f26
+oid sha256:d842c991349d997852c99ebf0dc6d368fc70b73c658d117a3088132a6cbb68ca
 size 2529523048

model_args.json

@@ -7,10 +7,8 @@
   "dim": 1024,
   "eos_id": 11,
   "ffn_dim": 3072,
-  "frame_sep_id": 233,
   "head_dim": 128,
   "image_cls_token_id": 244,
-  "image_mask_token_id": 243,
   "image_reg_1_token_id": 245,
   "image_reg_2_token_id": 246,
   "image_reg_3_token_id": 247,
@@ -35,7 +33,5 @@
   "size_token_id": 241,
   "spatial_patch_size": 16,
   "temporal_patch_size": 1,
-  "vid_end_id": 232,
-  "vid_start_id": 231,
   "vocab_size": 65536
 }

tokenizer.json

@@ -4752,7 +4752,32 @@
       }
     ]
   },
-  "post_processor": null,
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {}
+  },
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,

tokenizer_config.json

@@ -13,7 +13,26 @@
   "end_of_turn_token": "<|end_of_turn|>",
   "end_of_video_token": "<|end_of_video|>",
   "eos_token": "<|end_of_text|>",
-  "extra_special_tokens": {
+  "frame_sep_token": "<|frame_sep|>",
+  "image_cls_token": "<|image_cls|>",
+  "image_mask_token": "<|image_mask_token|>",
+  "image_reg_1_token": "<|image_reg_1|>",
+  "image_reg_2_token": "<|image_reg_2|>",
+  "image_reg_3_token": "<|image_reg_3|>",
+  "image_reg_4_token": "<|image_reg_4|>",
+  "image_reg_5_token": "<|image_reg_5|>",
+  "image_reg_6_token": "<|image_reg_6|>",
+  "image_reg_7_token": "<|image_reg_7|>",
+  "image_reg_8_token": "<|image_reg_8|>",
+  "image_row_sep_token": "<|image_row_sep|>",
+  "image_token": "<|image|>",
+  "is_local": true,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
     "absence_token": "<|absence|>",
     "caption_token": "<|CAPTION|>",
     "coord_token": "<|coord|>",
@@ -59,24 +78,6 @@
     "start_of_video_token": "<|start_of_video|>",
     "task_sep_token": "<|task_sep|>"
   },
-  "frame_sep_token": "<|frame_sep|>",
-  "image_cls_token": "<|image_cls|>",
-  "image_mask_token": "<|image_mask_token|>",
-  "image_reg_1_token": "<|image_reg_1|>",
-  "image_reg_2_token": "<|image_reg_2|>",
-  "image_reg_3_token": "<|image_reg_3|>",
-  "image_reg_4_token": "<|image_reg_4|>",
-  "image_reg_5_token": "<|image_reg_5|>",
-  "image_reg_6_token": "<|image_reg_6|>",
-  "image_reg_7_token": "<|image_reg_7|>",
-  "image_reg_8_token": "<|image_reg_8|>",
-  "image_row_sep_token": "<|image_row_sep|>",
-  "image_token": "<|image|>",
-  "model_input_names": [
-    "input_ids",
-    "attention_mask"
-  ],
-  "model_max_length": 1000000000000000019884624838656,
   "object_token": "<|object|>",
   "ocr_doc_parser_token": "<|OCR_DOC_PARSER|>",
   "ocr_grounding_token": "<|OCR_GROUNDING|>",
@@ -97,5 +98,5 @@
   "start_of_turn_token": "<|start_of_turn|>",
   "start_of_video_token": "<|start_of_video|>",
   "task_sep_token": "<|task_sep|>",
-  "tokenizer_class": "PreTrainedTokenizerFast"
+  "tokenizer_class": "TokenizersBackend"
 }