Upload 5 files

README.md

@@ -0,0 +1,64 @@
+---
+license: mit
+tags:
+- pytorch
+- tiny-transformer
+- retrieval
+---
+
+# Tiny Transformer for Retrieval
+
+## Overview
+
+A research-oriented **Tiny Transformer** prototype targeting **Retrieval**. The included **giant** setup documents defaults and file formats without presenting unverified performance numbers.
+
+## Repository status
+
+- The Python file contains the model and runnable example or training entry point.
+- `config.json` records the generated architecture settings.
+- `training_args.json` records the default experiment recipe.
+- `model.safetensors` is a valid initialization checkpoint for smoke tests; it is **not** presented as a trained benchmark checkpoint.
+- No benchmark score is claimed in this repository.
+
+## Architecture
+
+| Item | Value |
+|---|---|
+| Architecture | Tiny Transformer |
+| Scale | giant |
+| Attention | sparse |
+| Fusion | tensor fusion |
+| Activation | relu |
+| Normalization | layernorm |
+
+## Default experiment recipe
+
+The included configuration uses **lamb** with a **exponential** schedule. These are starting values in the script, not evidence of a completed run. For a meaningful evaluation, train all baselines with the same data exposure, tuning budget, and random seeds.
+
+## Quick check
+
+```bash
+python model.py --help
+```
+
+Inspect the script's `__main__` block for its generated smoke-test example. Because this is a custom implementation, generic automatic loading APIs require an explicit adapter before use.
+
+## Evaluation guidance
+
+A useful first evaluation would use **Flickr30k**, report the task metric across at least three seeds, and include a matched-capacity baseline. Keep training logs and environment versions with any published result.
+
+## Limitations
+
+The initialization checkpoint has not been trained or audited for robustness, fairness, or domain transfer. The implementation should be treated as an experimental starting point. Results from a future trained checkpoint must be documented separately from the defaults shipped here.
+
+## Files
+
+- `model.py` — primary artifact
+- `README.md` — this documentation
+- `config.json` — architecture configuration
+- `training_args.json` — default experiment settings
+- `model.safetensors` — initialization checkpoint
+
+## License
+
+Released under **mit**. Review the source-data terms separately when this repository is used with external datasets.

config.json

@@ -0,0 +1,16 @@
+{
+  "architectures": [
+    "CustomResearchModel"
+  ],
+  "architecture": "tiny_transformer",
+  "model_type": "tiny_transformer",
+  "hidden_size": 192,
+  "num_hidden_layers": 8,
+  "num_attention_heads": 8,
+  "intermediate_size": 768,
+  "hidden_act": "relu",
+  "max_position_embeddings": 512,
+  "layer_norm_eps": 1e-12,
+  "checkpoint_status": "initialization-only",
+  "notes": "Untrained checkpoint for smoke tests; no benchmark claim."
+}

model.py

@@ -0,0 +1,126 @@
+import torch
+import torch.nn as nn
+import math
+
+
+class TinyTransformerModel(nn.Module):
+    '''
+    tiny_transformer model with sparse attention.
+    Scale: giant (dim=768, layers=16, heads=12)
+    Fusion: tensor_fusion, Task: retrieval
+    '''
+
+    def __init__(self, embed_dim=768, num_layers=16, num_heads=12, num_classes=10):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.num_layers = num_layers
+        self.num_heads = num_heads
+
+        # image patch embedding
+        self.patch_proj = nn.Conv2d(3, embed_dim, kernel_size=16, stride=16)
+        self.cls_token = nn.Parameter(torch.randn(1, 1, embed_dim) * 0.02)
+        self.pos_embed = nn.Parameter(torch.randn(1, 197, embed_dim) * 0.02)
+        self.dropout = nn.Dropout(0.1)
+
+        # image transformer blocks
+        self.image_blocks = nn.ModuleList([
+            nn.TransformerEncoderLayer(
+                embed_dim, num_heads, embed_dim * 4, 0.1,
+                activation='gelu', batch_first=True, norm_first=True
+            )
+            for _ in range(num_layers)
+        ])
+        self.image_norm = nn.LayerNorm(embed_dim)
+
+        # text embedding
+        self.text_embed = nn.Embedding(30522, embed_dim, padding_idx=0)
+        self.text_pos = nn.Parameter(torch.randn(1, 128, embed_dim) * 0.02)
+        self.text_blocks = nn.ModuleList([
+            nn.TransformerEncoderLayer(
+                embed_dim, num_heads, embed_dim * 4, 0.1,
+                activation='gelu', batch_first=True, norm_first=True
+            )
+            for _ in range(num_layers)
+        ])
+        self.text_norm = nn.LayerNorm(embed_dim)
+
+        # fusion
+        self.fusion_blocks = nn.ModuleList([
+            nn.TransformerEncoderLayer(
+                embed_dim, num_heads, embed_dim * 4, 0.1,
+                activation='gelu', batch_first=True, norm_first=True
+            )
+            for _ in range(2)
+        ])
+        self.fusion_norm = nn.LayerNorm(embed_dim)
+
+        # task head
+        self.classifier = nn.Sequential(
+            nn.Linear(embed_dim, embed_dim),
+            nn.ReLU(),
+            nn.Dropout(0.1),
+            nn.Linear(embed_dim, num_classes),
+        )
+
+        self._initialize_weights()
+
+    def _initialize_weights(self):
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                nn.init.kaiming_normal_(m.weight)
+                if m.bias is not None:
+                    nn.init.zeros_(m.bias)
+            elif isinstance(m, nn.Embedding):
+                nn.init.trunc_normal_(m.weight, std=0.02)
+                if m.padding_idx is not None:
+                    m.weight[m.padding_idx].zero_()
+            elif isinstance(m, nn.LayerNorm):
+                nn.init.ones_(m.weight)
+                nn.init.zeros_(m.bias)
+
+    def encode_image(self, images):
+        x = self.patch_proj(images)
+        x = x.flatten(2).transpose(1, 2)
+        cls = self.cls_token.expand(x.size(0), -1, -1)
+        x = torch.cat([cls, x], dim=1)
+        x = x + self.pos_embed
+        x = self.dropout(x)
+        for block in self.image_blocks:
+            x = block(x)
+        return self.image_norm(x)
+
+    def encode_text(self, input_ids, attention_mask=None):
+        x = self.text_embed(input_ids)
+        x = x + self.text_pos[:, :input_ids.size(1)]
+        x = self.dropout(x)
+        padding = (attention_mask == 0) if attention_mask is not None else None
+        for block in self.text_blocks:
+            x = block(x, src_key_padding_mask=padding)
+        return self.text_norm(x)
+
+    def forward(self, images, input_ids, attention_mask=None, labels=None):
+        image_features = self.encode_image(images)
+        text_features = self.encode_text(input_ids, attention_mask)
+
+        fused = text_features
+        for block in self.fusion_blocks:
+            fused = block(fused)
+        fused = self.fusion_norm(fused[:, 0])
+        logits = self.classifier(fused)
+
+        loss = None
+        if labels is not None:
+            loss = nn.functional.cross_entropy(logits, labels)
+
+        return {'logits': logits, 'loss': loss}
+
+
+if __name__ == '__main__':
+    model = TinyTransformerModel()
+    total = sum(p.numel() for p in model.parameters())
+    print(f'TinyTransformerModel: {total:,} params ({total/1e6:.2f}M)')
+    img = torch.randn(2, 3, 224, 224)
+    ids = torch.randint(0, 30522, (2, 128))
+    mask = torch.ones(2, 128)
+    out = model(img, ids, mask, torch.tensor([0, 1]))
+    print(f'Output: {out["logits"].shape}, Loss: {out["loss"].item():.4f}')

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e2b0c4a390fdd9c7046947a2d04638201de8caa2429542dbe4344e9322876cc
+size 99808

training_args.json

@@ -0,0 +1,9 @@
+{
+  "optimizer": "lamb",
+  "scheduler": "exponential",
+  "learning_rate": 0.0001,
+  "batch_size": 24,
+  "epochs": 10,
+  "seed": 3407,
+  "status": "default recipe; not a completed run"
+}