KatieHg74 commited on
Commit
2cb2e8e
·
verified ·
1 Parent(s): ff0c1d8

Add model.py, README.md, config.json, training_args.json, model.safetensors

Browse files
Files changed (5) hide show
  1. README.md +84 -0
  2. config.json +19 -0
  3. model.py +126 -0
  4. model.safetensors +3 -0
  5. training_args.json +9 -0
README.md ADDED
@@ -0,0 +1,84 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ tags:
4
+ - pytorch
5
+ - efficientformer
6
+ - multitask
7
+ - wip
8
+ - initialization
9
+ ---
10
+
11
+ # Efficientformer for Multitask
12
+
13
+ ## Overview
14
+
15
+ This is an experimental **Efficientformer** codebase for **Multitask**. It keeps the **tiny** setup intentionally manageable so architecture changes can be inspected before a full training run.
16
+
17
+ ---
18
+
19
+ ## Architecture
20
+
21
+ - **Architecture**: Efficientformer
22
+ - **Scale**: tiny
23
+ - **Attention**: standard
24
+ - **Fusion**: cross attention
25
+ - **Activation**: relu
26
+ - **Normalization**: batchnorm
27
+
28
+ ---
29
+
30
+ ## Training details
31
+
32
+ - Weights initialized with a orthogonal scheme.
33
+ - Runs on a single consumer GPU.
34
+ - Full smoke test takes minutes, not hours.
35
+ - No distributed setup required.
36
+
37
+ ---
38
+
39
+ ## Limitations
40
+
41
+ The initialization checkpoint has not been trained or audited for robustness, fairness, or domain transfer. The implementation should be treated as an experimental starting point. Results from a future trained checkpoint must be documented separately from the defaults shipped here.
42
+
43
+ ---
44
+
45
+ ## Default experiment recipe
46
+
47
+ The included configuration uses **lion** with a **linear warmup** schedule. These are starting values in the script, not evidence of a completed run. For a meaningful evaluation, train all baselines with the same data exposure, tuning budget, and random seeds.
48
+
49
+ ---
50
+
51
+ ## Quick check
52
+
53
+ Run `python model.py --help` for the built-in smoke test. Because this is a custom implementation, generic automatic loading APIs require an explicit adapter before use.
54
+
55
+ ---
56
+
57
+ ## Roadmap
58
+
59
+ Next steps, in no particular order: a cleaner entry point, input-shape docs, and a small evaluation script if time permits.
60
+
61
+ ---
62
+
63
+ ## Repository status
64
+
65
+ One Python module contains the model and a runnable example.
66
+ Generated configuration lives in `config.json`; the recipe in `training_args.json`.
67
+ The checkpoint is an initialization artifact and should not be read as a trained model.
68
+ Scores are deliberately absent.
69
+
70
+ ---
71
+
72
+ ## Files
73
+
74
+ - `model.py` — primary artifact
75
+ - `README.md` — this documentation
76
+ - `config.json` — architecture configuration
77
+ - `training_args.json` — default experiment settings
78
+ - `model.safetensors` — initialization checkpoint
79
+
80
+ ---
81
+
82
+ ## License
83
+
84
+ Released under **apache-2.0**. Review the source-data terms separately when this repository is used with external datasets.
config.json ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "CustomResearchModel"
4
+ ],
5
+ "architecture": "efficientformer",
6
+ "model_type": "efficientformer",
7
+ "hidden_size": 256,
8
+ "num_hidden_layers": 8,
9
+ "num_attention_heads": 8,
10
+ "intermediate_size": 512,
11
+ "hidden_act": "relu",
12
+ "max_position_embeddings": 256,
13
+ "layer_norm_eps": 1e-12,
14
+ "checkpoint_status": "initialization-only",
15
+ "notes": "Untrained checkpoint for smoke tests; no benchmark claim.",
16
+ "use_cache": true,
17
+ "torch_dtype": "float32",
18
+ "vocab_size": 2048
19
+ }
model.py ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import torch
2
+ import torch.nn as nn
3
+ import math
4
+
5
+
6
+ class EfficientformerModel(nn.Module):
7
+ '''
8
+ efficientformer model with standard attention.
9
+ Scale: tiny (dim=128, layers=3, heads=2)
10
+ Fusion: cross_attention, Task: multitask
11
+ '''
12
+
13
+ def __init__(self, embed_dim=128, num_layers=3, num_heads=2, num_classes=10):
14
+ super().__init__()
15
+ self.embed_dim = embed_dim
16
+ self.num_layers = num_layers
17
+ self.num_heads = num_heads
18
+
19
+ # image patch embedding
20
+ self.patch_proj = nn.Conv2d(3, embed_dim, kernel_size=16, stride=16)
21
+ self.cls_token = nn.Parameter(torch.randn(1, 1, embed_dim) * 0.02)
22
+ self.pos_embed = nn.Parameter(torch.randn(1, 197, embed_dim) * 0.02)
23
+ self.dropout = nn.Dropout(0.1)
24
+
25
+ # image transformer blocks
26
+ self.image_blocks = nn.ModuleList([
27
+ nn.TransformerEncoderLayer(
28
+ embed_dim, num_heads, embed_dim * 4, 0.1,
29
+ activation='gelu', batch_first=True, norm_first=True
30
+ )
31
+ for _ in range(num_layers)
32
+ ])
33
+ self.image_norm = nn.LayerNorm(embed_dim)
34
+
35
+ # text embedding
36
+ self.text_embed = nn.Embedding(30522, embed_dim, padding_idx=0)
37
+ self.text_pos = nn.Parameter(torch.randn(1, 128, embed_dim) * 0.02)
38
+ self.text_blocks = nn.ModuleList([
39
+ nn.TransformerEncoderLayer(
40
+ embed_dim, num_heads, embed_dim * 4, 0.1,
41
+ activation='gelu', batch_first=True, norm_first=True
42
+ )
43
+ for _ in range(num_layers)
44
+ ])
45
+ self.text_norm = nn.LayerNorm(embed_dim)
46
+
47
+ # fusion
48
+ self.fusion_blocks = nn.ModuleList([
49
+ nn.TransformerEncoderLayer(
50
+ embed_dim, num_heads, embed_dim * 4, 0.1,
51
+ activation='gelu', batch_first=True, norm_first=True
52
+ )
53
+ for _ in range(2)
54
+ ])
55
+ self.fusion_norm = nn.LayerNorm(embed_dim)
56
+
57
+ # task head
58
+ self.classifier = nn.Sequential(
59
+ nn.Linear(embed_dim, embed_dim),
60
+ nn.ReLU(),
61
+ nn.Dropout(0.1),
62
+ nn.Linear(embed_dim, num_classes),
63
+ )
64
+
65
+ self._initialize_weights()
66
+
67
+ def _initialize_weights(self):
68
+ for m in self.modules():
69
+ if isinstance(m, nn.Linear):
70
+ nn.init.orthogonal_(m.weight)
71
+ if m.bias is not None:
72
+ nn.init.zeros_(m.bias)
73
+ elif isinstance(m, nn.Embedding):
74
+ nn.init.trunc_normal_(m.weight, std=0.02)
75
+ if m.padding_idx is not None:
76
+ m.weight[m.padding_idx].zero_()
77
+ elif isinstance(m, nn.LayerNorm):
78
+ nn.init.ones_(m.weight)
79
+ nn.init.zeros_(m.bias)
80
+
81
+ def encode_image(self, images):
82
+ x = self.patch_proj(images)
83
+ x = x.flatten(2).transpose(1, 2)
84
+ cls = self.cls_token.expand(x.size(0), -1, -1)
85
+ x = torch.cat([cls, x], dim=1)
86
+ x = x + self.pos_embed
87
+ x = self.dropout(x)
88
+ for block in self.image_blocks:
89
+ x = block(x)
90
+ return self.image_norm(x)
91
+
92
+ def encode_text(self, input_ids, attention_mask=None):
93
+ x = self.text_embed(input_ids)
94
+ x = x + self.text_pos[:, :input_ids.size(1)]
95
+ x = self.dropout(x)
96
+ padding = (attention_mask == 0) if attention_mask is not None else None
97
+ for block in self.text_blocks:
98
+ x = block(x, src_key_padding_mask=padding)
99
+ return self.text_norm(x)
100
+
101
+ def forward(self, images, input_ids, attention_mask=None, labels=None):
102
+ image_features = self.encode_image(images)
103
+ text_features = self.encode_text(input_ids, attention_mask)
104
+
105
+ fused = text_features
106
+ for block in self.fusion_blocks:
107
+ fused = block(fused)
108
+ fused = self.fusion_norm(fused[:, 0])
109
+ logits = self.classifier(fused)
110
+
111
+ loss = None
112
+ if labels is not None:
113
+ loss = nn.functional.cross_entropy(logits, labels)
114
+
115
+ return {'logits': logits, 'loss': loss}
116
+
117
+
118
+ if __name__ == '__main__':
119
+ model = EfficientformerModel()
120
+ total = sum(p.numel() for p in model.parameters())
121
+ print(f'EfficientformerModel: {total:,} params ({total/1e6:.2f}M)')
122
+ img = torch.randn(2, 3, 224, 224)
123
+ ids = torch.randint(0, 30522, (2, 128))
124
+ mask = torch.ones(2, 128)
125
+ out = model(img, ids, mask, torch.tensor([0, 1]))
126
+ print(f'Output: {out["logits"].shape}, Loss: {out["loss"].item():.4f}')
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c8331ffaf0be36c3426e5a204a373fc409afc4932a70fb66f2a396730ffd7c3
3
+ size 132832
training_args.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "optimizer": "lion",
3
+ "scheduler": "linear_warmup",
4
+ "learning_rate": 0.0001,
5
+ "batch_size": 16,
6
+ "epochs": 30,
7
+ "seed": 42,
8
+ "status": "default recipe; not a completed run"
9
+ }