Upload genuine_model.py with huggingface_hub

genuine_model.py

@@ -109,13 +109,13 @@ class GenuinenessGate(nn.Module):
         return self.gate_fc(feat)
 
 class GenuineTransformer(nn.Module):
-    def __init__(self, d_model=256, n_heads=8, n_layers=6, vocab_size=1000):
+    def __init__(self, d_model=512, n_heads=8, n_layers=12, vocab_size=1000):
         super().__init__()
         self.embedding = nn.Embedding(vocab_size, d_model)
         self.layers = nn.ModuleList([GenuineLayer(d_model, n_heads) for _ in range(n_layers)])
         self.gate = GenuinenessGate(d_model, n_heads)
         self.fc_out = nn.Linear(d_model, vocab_size)
-        self.register_buffer("freqs_cis", precompute_freqs_cis(d_model // n_heads, 128))
+        self.register_buffer("freqs_cis", precompute_freqs_cis(d_model // n_heads, 256))
         self.n_heads = n_heads
 
     def forward(self, x, g_budget=12):
@@ -125,26 +125,26 @@ class GenuineTransformer(nn.Module):
         x = self.embedding(x)
         all_entropies = []
         reasoning_layers = len(self.layers) // 2
-        
+
         # V2.2: Global G-Budgeting
         total_steps = 0
-        
+
         while total_steps < g_budget:
             loop_entropies = []
             for i in range(reasoning_layers):
                 x, attn, entropies = self.layers[i](x, freqs_cis)
                 loop_entropies.append(entropies)
                 total_steps += 1
-            
+
             all_entropies.extend(loop_entropies)
-            
+
             # Gating check
             last_entropy = loop_entropies[-1]
             gate_signal = self.gate(x, last_entropy) # [batch, 1]
-            
+
             if gate_signal.mean() < 0.5:
                 break
-                
+
         # Final decoding layers
         for i in range(reasoning_layers, len(self.layers)):
             x, attn, entropies = self.layers[i](x, freqs_cis)
@@ -173,11 +173,11 @@ class ThermodynamicRegularizer:
         # 1. Variance Reward (Internal Complexity)
         # Reward high entropy variance across heads for each token/layer
         var_h = torch.var(stack, dim=-1) # [layers, batch, seq]
-        
+
         # Apply layer-wise decay to prioritize early-layer complexity
         layer_weights = torch.pow(self.layer_decay, torch.arange(len(entropies), device=stack.device, dtype=torch.float)).view(-1, 1, 1)
         weighted_var = (var_h * layer_weights).mean()
-        
+
         total_loss = -self.variance_weight * weighted_var
 
         # 2. Mechanical Penalty (Anti-Pattern Matching)