Upload folder using huggingface_hub

.gitattributes

@@ -243,3 +243,13 @@ arc1-Gym-G_24/step_25933 filter=lfs diff=lfs merge=lfs -text
 arc1-Gym-G_24/step_259332 filter=lfs diff=lfs merge=lfs -text
 arc1-Gym-G_24/step_51866 filter=lfs diff=lfs merge=lfs -text
 arc1-Gym-G_24/step_77800 filter=lfs diff=lfs merge=lfs -text
+mini-GC/step_10304 filter=lfs diff=lfs merge=lfs -text
+mini-GC/step_11592 filter=lfs diff=lfs merge=lfs -text
+mini-GC/step_1288 filter=lfs diff=lfs merge=lfs -text
+mini-GC/step_12880 filter=lfs diff=lfs merge=lfs -text
+mini-GC/step_2576 filter=lfs diff=lfs merge=lfs -text
+mini-GC/step_3864 filter=lfs diff=lfs merge=lfs -text
+mini-GC/step_5152 filter=lfs diff=lfs merge=lfs -text
+mini-GC/step_6440 filter=lfs diff=lfs merge=lfs -text
+mini-GC/step_7728 filter=lfs diff=lfs merge=lfs -text
+mini-GC/step_9016 filter=lfs diff=lfs merge=lfs -text

mini-GC/all_config.yaml

@@ -0,0 +1,52 @@
+arch:
+  ACT_threshold: 0
+  H_cycles: 3
+  H_layers: 0
+  L_cycles: 4
+  L_layers: 2
+  expansion: 4
+  forward_dtype: bfloat16
+  group_emb_len: 1
+  halt_exploration_prob: 0.1
+  halt_max_steps: 16
+  hidden_size: 512
+  loss:
+    alpha: 0.5
+    loss_type: stablemax_cross_entropy
+    name: losses@ACTLossHead
+  mlp_t: false
+  name: recursive_reasoning.trm@TinyRecursiveReasoningModel_ACTV1
+  no_ACT_continue: true
+  num_heads: 8
+  pos_encodings: rope
+  puzzle_emb_len: 16
+  puzzle_emb_ndim: 512
+  universal_emb_len: 1
+beta1: 0.9
+beta2: 0.95
+checkpoint_every_eval: true
+checkpoint_path: ./checkpoint/miniarc/mini_GC_losses@ACTLossHead
+data_paths:
+- data/miniARC-aug-1000
+data_paths_test: []
+ema: true
+ema_rate: 0.999
+epochs: 100000
+eval_interval: 10000
+eval_save_outputs: []
+evaluators: []
+freeze_weights: false
+global_batch_size: 4096
+group_emb_lr: 0.001
+group_emb_weight_decay: 0.1
+load_checkpoint: null
+lr: 0.0001
+lr_min_ratio: 1.0
+lr_warmup_steps: 2000
+min_eval_interval: 0
+project_name: Miniarc-aug-1000-ACT-torch
+puzzle_emb_lr: 0.01
+puzzle_emb_weight_decay: 0.1
+run_name: mini_GC_losses@ACTLossHead
+seed: 0
+weight_decay: 0.1

mini-GC/losses.py

@@ -0,0 +1,192 @@
+from typing import Any, Tuple, Dict, Sequence, Optional
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+import math
+
+IGNORE_LABEL_ID = -100
+
+
+def s(x, epsilon=1e-30):
+    return torch.where(
+        x<0,
+        1/(1-x+ epsilon),
+        x + 1
+    )
+
+
+def log_stablemax(x, dim=-1):
+    s_x = s(x)
+    return torch.log(s_x/torch.sum(s_x, dim=dim, keepdim=True))
+
+
+def stablemax_cross_entropy(logits, labels, ignore_index: int = -100, valid_mask=None):
+    logprobs = log_stablemax(logits.to(torch.float64), dim=-1)
+
+    if valid_mask is None:
+        valid_mask = (labels != ignore_index)
+    transformed_labels = torch.where(valid_mask, labels, 0)
+    prediction_logprobs = torch.gather(logprobs, index=transformed_labels.to(torch.long).unsqueeze(-1), dim=-1).squeeze(-1)
+
+    return -torch.where(valid_mask, prediction_logprobs, 0)
+
+
+def softmax_cross_entropy(logits, labels, ignore_index: int = -100):
+    # Cast logits to f32
+    # Flatten logits
+    return F.cross_entropy(logits.to(torch.float32).view(-1, logits.shape[-1]), labels.to(torch.long).view(-1), ignore_index=ignore_index, reduction="none").view(labels.shape)
+
+
+class ACTLossHead(nn.Module):
+    def __init__(self, model: nn.Module, loss_type: str, alpha: float = 0.5):
+        super().__init__()
+        self.model = model
+        self.loss_fn = globals()[loss_type]
+        
+    def initial_carry(self, *args, **kwargs):
+        return self.model.initial_carry(*args, **kwargs)  # type: ignore
+    
+    @staticmethod
+    def comput_q_halt_loss(self, outputs, is_correct, seq_is_correct, loss_counts):
+        q_halt_loss = F.binary_cross_entropy_with_logits(outputs["q_halt_logits"], seq_is_correct.to(outputs["q_halt_logits"].dtype), reduction="sum")
+        return q_halt_loss
+
+    def forward(
+        self,
+        return_keys: Sequence[str],
+        # Model args
+        **model_kwargs,
+    ) -> Tuple[Any, torch.Tensor, Dict[str, torch.Tensor], Optional[Dict[str, torch.Tensor]], torch.Tensor]:
+        # Model logits
+        # B x SeqLen x D
+        new_carry, outputs = self.model(**model_kwargs)
+        labels = new_carry.current_data["labels"]
+
+        with torch.no_grad():
+            # Preds
+            outputs["preds"] = torch.argmax(outputs["logits"], dim=-1)
+
+            # Correctness
+            mask = (labels != IGNORE_LABEL_ID)
+            loss_counts = mask.sum(-1)
+            loss_divisor = loss_counts.clamp_min(1).unsqueeze(-1)  # Avoid NaNs in division
+
+            is_correct = mask & (torch.argmax(outputs["logits"], dim=-1) == labels)
+            seq_is_correct = is_correct.sum(-1) == loss_counts
+            
+            # Metrics (halted)
+            valid_metrics = new_carry.halted & (loss_counts > 0)
+            metrics = {
+                "count": valid_metrics.sum(),
+                
+                "accuracy":       torch.where(valid_metrics, (is_correct.to(torch.float32) / loss_divisor).sum(-1), 0).sum(),
+                "exact_accuracy": (valid_metrics & seq_is_correct).sum(),
+
+                "q_halt_accuracy": (valid_metrics & ((outputs["q_halt_logits"] >= 0) == seq_is_correct)).sum(),
+                "steps":          torch.where(valid_metrics, new_carry.steps, 0).sum(),
+            }
+
+        # Losses
+        lm_loss = (self.loss_fn(outputs["logits"], labels, ignore_index=IGNORE_LABEL_ID, valid_mask=mask) / loss_divisor).sum()
+        q_halt_loss = self.comput_q_halt_loss(self, outputs, is_correct, seq_is_correct, loss_counts)
+        metrics.update({
+            "lm_loss": lm_loss.detach(),
+            "q_halt_loss": q_halt_loss.detach(),
+        })
+        # Q continue (bootstrapping target loss); Alexia: This fits Q-learning, but seems totally unecessary
+        q_continue_loss = 0
+        if "target_q_continue" in outputs:
+            q_continue_loss = F.binary_cross_entropy_with_logits(outputs["q_continue_logits"], outputs["target_q_continue"], reduction="sum")
+
+            metrics["q_continue_loss"] = q_continue_loss.detach()
+        # Filter outputs for return
+        detached_outputs = {k: outputs[k].detach() for k in return_keys if k in outputs}
+
+        return new_carry, lm_loss + 0.5 * (q_halt_loss + q_continue_loss), metrics, detached_outputs, new_carry.halted.all()
+
+
+class ACTLossHead_binorminal(ACTLossHead):
+    def __init__(self, model: nn.Module, loss_type: str, alpha: float = 0.5):
+        super().__init__(model, loss_type)
+        
+    @staticmethod
+    def comput_q_halt_loss(self, outputs, is_correct, seq_is_correct, loss_counts):
+        t = (is_correct.sum(-1) / loss_counts.clamp_min(1)).to(outputs["q_halt_logits"].dtype)  # [B] in [0,1]
+
+        # Force t=1 for sequences with no valid tokens
+        t = torch.where(loss_counts == 0, torch.ones_like(t), t)
+        q_halt_loss = F.binary_cross_entropy_with_logits(
+            outputs["q_halt_logits"], t, reduction="sum"
+        )        
+        return q_halt_loss
+
+class ACTLossHead_combine(ACTLossHead):
+    def __init__(self, model: nn.Module, loss_type: str, alpha: float = 0.5):
+        super().__init__(model, loss_type)
+        self.alpha = alpha
+
+    @staticmethod
+    def comput_q_halt_loss(self, outputs, is_correct, seq_is_correct, loss_counts):
+        q_halt_bce_loss = ACTLossHead.comput_q_halt_loss(self, outputs, is_correct, seq_is_correct, loss_counts)
+        q_halt_binorm_loss = ACTLossHead_binorminal.comput_q_halt_loss(self, outputs, is_correct, seq_is_correct, loss_counts)
+        q_halt_loss = self.alpha * q_halt_bce_loss + (1 - self.alpha) * q_halt_binorm_loss
+        return q_halt_loss
+
+
+class ACTLossHead_binormonly(ACTLossHead_binorminal):
+    def __init__(self, model: nn.Module, loss_type: str):
+        super().__init__(model, loss_type)
+
+    def forward(
+        self,
+        return_keys: Sequence[str],
+        # Model args
+        **model_kwargs,
+    ) -> Tuple[Any, torch.Tensor, Dict[str, torch.Tensor], Optional[Dict[str, torch.Tensor]], torch.Tensor]:
+        # Model logits
+        # B x SeqLen x D
+        new_carry, outputs = self.model(**model_kwargs)
+        labels = new_carry.current_data["labels"]
+
+        with torch.no_grad():
+            # Preds
+            outputs["preds"] = torch.argmax(outputs["logits"], dim=-1)
+
+            # Correctness
+            mask = (labels != IGNORE_LABEL_ID)
+            loss_counts = mask.sum(-1)
+            loss_divisor = loss_counts.clamp_min(1).unsqueeze(-1)  # Avoid NaNs in division
+
+            is_correct = mask & (torch.argmax(outputs["logits"], dim=-1) == labels)
+            seq_is_correct = is_correct.sum(-1) == loss_counts
+            
+            # Metrics (halted)
+            valid_metrics = new_carry.halted & (loss_counts > 0)
+            metrics = {
+                "count": valid_metrics.sum(),
+                
+                "accuracy":       torch.where(valid_metrics, (is_correct.to(torch.float32) / loss_divisor).sum(-1), 0).sum(),
+                "exact_accuracy": (valid_metrics & seq_is_correct).sum(),
+
+                "q_halt_accuracy": (valid_metrics & ((outputs["q_halt_logits"] >= 0) == seq_is_correct)).sum(),
+                "steps":          torch.where(valid_metrics, new_carry.steps, 0).sum(),
+            }
+
+        # Losses
+        lm_loss = (self.loss_fn(outputs["logits"], labels, ignore_index=IGNORE_LABEL_ID, valid_mask=mask) / loss_divisor).sum()
+        q_halt_loss = self.comput_q_halt_loss(self, outputs, is_correct, seq_is_correct, loss_counts)
+        metrics.update({
+            "lm_loss": lm_loss.detach(),
+            "q_halt_loss": q_halt_loss.detach(),
+        })
+        # Q continue (bootstrapping target loss); Alexia: This fits Q-learning, but seems totally unecessary
+        q_continue_loss = 0
+        if "target_q_continue" in outputs:
+            q_continue_loss = F.binary_cross_entropy_with_logits(outputs["q_continue_logits"], outputs["target_q_continue"], reduction="sum")
+
+            metrics["q_continue_loss"] = q_continue_loss.detach()
+        # Filter outputs for return
+        detached_outputs = {k: outputs[k].detach() for k in return_keys if k in outputs}
+
+        return new_carry,  (q_halt_loss + q_continue_loss), metrics, detached_outputs, new_carry.halted.all()

mini-GC/step_10304

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d17091f5de2135661325fe3fbb972c340e4aa0ddf28bb75045402e9ff9aceb63
+size 270927925

mini-GC/step_11592

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ed5cb340c8ae0137b5e40545123d21b8b282fc18cf0ea98530fa479aeacc49b
+size 270927925

mini-GC/step_1288

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4922745deb401c0a2447223ae9fde8ece3fbf5bbe2681ebc93aa182dbcaab244
+size 270927902

mini-GC/step_12880

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e366e61f3bd49e95e0f41a57fa922e56fff129b1bec0ac9fd00328acd49500f
+size 270927925

mini-GC/step_2576

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c398e2ad28b143db9a0f21ff085c3f748dd0bf2b084f5ab933310108bc8bc47b
+size 270927902

mini-GC/step_3864

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:667a203fcd93a03fc3154a5f59e4b2c4ab23ff7340306cb03461d0d17f87ff33
+size 270927902

mini-GC/step_5152

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bd545e486f722cb4c359ca78f127b27123cb21b840cf9ddf9d3183cdd8f5892
+size 270927902

mini-GC/step_6440

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca4b025dd6b276bc191d53b4ed0ebb9da40f2a7edd28d1cff3a7725df46bf5d6
+size 270927902

mini-GC/step_7728

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d40aac6ab02626026b4703b18ae42a66ebfbcfd87a45411a77f3334a2749d93f
+size 270927902

mini-GC/step_9016

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0521aeb7f5778e7f1d4879b35de69a3e045071b5b0077551005344ca7f8e2c8
+size 270927902

mini-GC/trm.py

@@ -0,0 +1,445 @@
+from typing import Tuple, List, Dict, Optional
+from dataclasses import dataclass
+import math
+import torch
+import copy
+import torch.nn.functional as F
+from torch import nn
+from pydantic import BaseModel
+import random
+from models.common import trunc_normal_init_
+from models.layers import rms_norm, LinearSwish, SwiGLU, Attention, RotaryEmbedding, CosSin, CastedEmbedding, CastedLinear
+from models.sparse_embedding import CastedSparseEmbedding
+
+IGNORE_LABEL_ID = -100
+
+@dataclass
+class TinyRecursiveReasoningModel_ACTV1InnerCarry:
+    z_H: torch.Tensor
+    z_L: torch.Tensor
+
+
+@dataclass
+class TinyRecursiveReasoningModel_ACTV1Carry:
+    inner_carry: TinyRecursiveReasoningModel_ACTV1InnerCarry
+    
+    steps: torch.Tensor
+    halted: torch.Tensor
+    
+    current_data: Dict[str, torch.Tensor]
+
+
+class TinyRecursiveReasoningModel_ACTV1Config(BaseModel):
+    batch_size: int
+    seq_len: int
+    puzzle_emb_ndim: int = 0
+    universal_emb_len: int = 0
+    group_emb_len: int = 0
+    num_puzzle_identifiers: int
+    num_groups: int
+    vocab_size: int
+
+    H_cycles: int
+    L_cycles: int
+
+    H_layers: int # ignored
+    L_layers: int
+
+    # Transformer config
+    hidden_size: int
+    expansion: float
+    num_heads: int
+    pos_encodings: str
+
+    rms_norm_eps: float = 1e-5
+    rope_theta: float = 10000.0
+    
+    # Halting Q-learning config
+    halt_max_steps: int
+    halt_exploration_prob: float
+    puzzle_emb_len: int = 16 # if non-zero, its specified to this value
+    forward_dtype: str = "bfloat16"
+
+    # Alexia: added
+    mlp_t: bool = False # use mlp on L instead of transformer
+    no_ACT_continue: bool =  True # No continue ACT loss, only use the sigmoid of the halt which makes much more sense
+
+    # chan edit
+    eval_with_ACT: bool = False # During evaluation, whether to use ACT halting or just run max steps. Using ACT halting will cause more variance in evaluation results, but is a more realistic evaluation of the model performance when deployed.
+    ACT_threshold: float = 0 # Threshold for ACT halting during evaluation, only used if eval_with_ACT is True
+
+class TinyRecursiveReasoningModel_ACTGymConfig(BaseModel):
+    replace_halt_threshold: int = 1
+    target_latent_for_reinit: str = "L"
+
+class TinyRecursiveReasoningModel_ACTGym(BaseModel):
+    """ACT wrapper."""
+
+    def __init__(self, config_dict: dict):
+        super().__init__(config_dict)
+        self.replace_halt_threshold: int = 1
+        self.target_latent_for_reinit: str = "L"
+
+class TinyRecursiveReasoningModel_ACTV1Block(nn.Module):
+    def __init__(self, config: TinyRecursiveReasoningModel_ACTV1Config, puzzle_emb_len: int) -> None:
+        super().__init__()
+
+        self.config = config
+        if self.config.mlp_t:
+            self.puzzle_emb_len = puzzle_emb_len
+            self.mlp_t = SwiGLU(
+                hidden_size=self.config.seq_len + self.puzzle_emb_len, # L
+                expansion=config.expansion,
+            )
+        else:
+            self.self_attn = Attention(
+                hidden_size=config.hidden_size,
+                head_dim=config.hidden_size // config.num_heads,
+                num_heads=config.num_heads,
+                num_key_value_heads=config.num_heads,
+                causal=False
+            )
+        self.mlp = SwiGLU(
+            hidden_size=config.hidden_size,
+            expansion=config.expansion,
+        )
+        self.norm_eps = config.rms_norm_eps
+
+    def forward(self, cos_sin: CosSin, hidden_states: torch.Tensor) -> torch.Tensor:
+        # B, L, D = hidden_states.shape
+        # Post Norm
+        if self.config.mlp_t:
+            hidden_states = hidden_states.transpose(1,2)
+            out = self.mlp_t(hidden_states)
+            hidden_states = rms_norm(hidden_states + out, variance_epsilon=self.norm_eps)
+            hidden_states = hidden_states.transpose(1,2)
+        else:
+            # Self Attention
+            hidden_states = rms_norm(hidden_states + self.self_attn(cos_sin=cos_sin, hidden_states=hidden_states), variance_epsilon=self.norm_eps)
+        # Fully Connected
+        out = self.mlp(hidden_states)
+        hidden_states = rms_norm(hidden_states + out, variance_epsilon=self.norm_eps)
+        return hidden_states
+
+class TinyRecursiveReasoningModel_ACTV1ReasoningModule(nn.Module):
+    def __init__(self, layers: List[TinyRecursiveReasoningModel_ACTV1Block]):
+        super().__init__()
+        self.layers = torch.nn.ModuleList(layers)
+
+    def forward(self, hidden_states: torch.Tensor, input_injection: torch.Tensor, **kwargs) -> torch.Tensor:
+        hidden_states = hidden_states + input_injection
+        for layer in self.layers:
+            hidden_states = layer(hidden_states=hidden_states, **kwargs)
+        return hidden_states
+
+
+class TinyRecursiveReasoningModel_ACTV1_Inner(nn.Module):
+    def __init__(self, config: TinyRecursiveReasoningModel_ACTV1Config) -> None:
+        super().__init__()
+        self.config = config
+        self.forward_dtype = getattr(torch, self.config.forward_dtype)
+
+        # I/O
+
+        self.embed_scale = math.sqrt(self.config.hidden_size)
+        embed_init_std = 1.0 / self.embed_scale
+
+        self.embed_tokens = CastedEmbedding(self.config.vocab_size, self.config.hidden_size, init_std=embed_init_std, cast_to=self.forward_dtype)
+        self.lm_head      = CastedLinear(self.config.hidden_size, self.config.vocab_size, bias=False)
+        self.q_head       = CastedLinear(self.config.hidden_size, 2, bias=True)
+
+        if self.config.puzzle_emb_ndim > 0:
+            # Zero init puzzle embeddings
+            self.puzzle_emb = CastedSparseEmbedding(self.config.num_puzzle_identifiers, self.config.puzzle_emb_ndim,
+                                                    batch_size=self.config.batch_size, init_std=0, cast_to=self.forward_dtype)
+        
+        self.group_embedding_dim = self.config.group_emb_len * self.config.hidden_size
+        if self.group_embedding_dim > 0:
+            # Zero init group embeddings
+            self.group_emb = CastedSparseEmbedding(self.config.num_groups, self.group_embedding_dim,
+                                                    batch_size=self.config.batch_size, init_std=0, cast_to=self.forward_dtype)
+
+        self.universal_emb_ndim = self.config.universal_emb_len * self.config.hidden_size
+        if self.universal_emb_ndim > 0: 
+            # Universal embedding, shared between tokens and puzzles, with non-zero init
+            self.universal_emb = CastedEmbedding(1, self.universal_emb_ndim, init_std=embed_init_std, cast_to=self.forward_dtype)
+        
+        print("Puzzle embedding ndim:", self.config.puzzle_emb_ndim)
+        print("Group embedding ndim:", self.group_embedding_dim)
+        print("Universal embedding ndim:", self.universal_emb_ndim)
+        print("Puzzle embed length:", self.config.puzzle_emb_len)
+        total_puzzle_emb_dim = self.config.puzzle_emb_ndim + self.group_embedding_dim + self.universal_emb_ndim
+        self.puzzle_emb_len = max(-(total_puzzle_emb_dim // -self.config.hidden_size),  self.config.puzzle_emb_len)  # ceil div
+        print("Total puzzle embedding dim:", total_puzzle_emb_dim)
+        print("Final puzzle embedding length (in tokens):", self.puzzle_emb_len)
+
+        # LM Blocks
+        if self.config.pos_encodings == "rope":
+            self.rotary_emb = RotaryEmbedding(dim=self.config.hidden_size // self.config.num_heads,
+                                              max_position_embeddings=self.config.seq_len + self.puzzle_emb_len,
+                                              base=self.config.rope_theta)
+        elif self.config.pos_encodings == "learned":
+            self.embed_pos = CastedEmbedding(self.config.seq_len + self.puzzle_emb_len, self.config.hidden_size, init_std=embed_init_std, cast_to=self.forward_dtype)
+        else:
+            pass
+
+        # Reasoning Layers
+        self.L_level = TinyRecursiveReasoningModel_ACTV1ReasoningModule(layers=[TinyRecursiveReasoningModel_ACTV1Block(self.config, self.puzzle_emb_len) for _i in range(self.config.L_layers)])
+
+        # Initial states
+        self.H_init = self.init_latent_buffer(self.config.hidden_size, self.forward_dtype)
+        self.L_init = self.init_latent_buffer(self.config.hidden_size, self.forward_dtype)
+
+        # Q head special init
+        # Init Q to (almost) zero for faster learning during bootstrapping
+        with torch.no_grad():
+            self.q_head.weight.zero_()
+            self.q_head.bias.fill_(-5)  # type: ignore
+
+    @staticmethod
+    def init_latent_buffer(
+        hidden_size: int,
+        forward_dtype: torch.dtype,
+        persistent: bool = True,
+        device: Optional[torch.device] = None,
+    ) -> torch.Tensor:
+        return nn.Buffer(
+            trunc_normal_init_(torch.empty(hidden_size, dtype=forward_dtype, device=device), std=1),
+            persistent=persistent,
+        )
+
+    def _input_embeddings(self, input: torch.Tensor, puzzle_identifiers: torch.Tensor, group_indices: torch.Tensor):
+        # Token embedding
+        embedding = self.embed_tokens(input.to(torch.int32))
+
+        # Puzzle embeddings
+        if self.config.puzzle_emb_ndim > 0:
+            puzzle_embedding = self.puzzle_emb(puzzle_identifiers)
+
+            if self.config.group_emb_len > 0:
+                group_embedding = self.group_emb(group_indices)
+
+                # concat along feature dim -> (B, Dp + Dg)
+                puzzle_embedding = torch.cat([puzzle_embedding, group_embedding], dim=-1)
+
+            if self.config.universal_emb_len > 0:
+                # universal_embedding: (B, Du)
+                universal_embedding = self.universal_emb(torch.zeros(puzzle_embedding.size(0), device=puzzle_embedding.device, dtype=torch.long))
+
+                # concat along feature dim -> (B, Dp + (Dg) + Du)
+                puzzle_embedding = torch.cat([puzzle_embedding, universal_embedding], dim=-1)
+                
+            pad_count = self.puzzle_emb_len * self.config.hidden_size - puzzle_embedding.shape[-1]
+            if pad_count > 0:
+                puzzle_embedding = F.pad(puzzle_embedding, (0, pad_count))
+
+            embedding = torch.cat((puzzle_embedding.view(-1, self.puzzle_emb_len, self.config.hidden_size), embedding), dim=-2)
+           
+        # Position embeddings
+        if self.config.pos_encodings == "learned":
+            # scale by 1/sqrt(2) to maintain forward variance
+            embedding = 0.707106781 * (embedding + self.embed_pos.embedding_weight.to(self.forward_dtype))
+
+        # Scale
+        return self.embed_scale * embedding
+
+    def empty_carry(self, batch_size: int):
+        return TinyRecursiveReasoningModel_ACTV1InnerCarry(
+            z_H=torch.empty(batch_size, self.config.seq_len + self.puzzle_emb_len, self.config.hidden_size, dtype=self.forward_dtype),
+            z_L=torch.empty(batch_size, self.config.seq_len + self.puzzle_emb_len, self.config.hidden_size, dtype=self.forward_dtype),
+        )
+        
+    def reset_carry(self, reset_flag: torch.Tensor, carry: TinyRecursiveReasoningModel_ACTV1InnerCarry):
+        return TinyRecursiveReasoningModel_ACTV1InnerCarry(
+            z_H=torch.where(reset_flag.view(-1, 1, 1), self.H_init, carry.z_H),
+            z_L=torch.where(reset_flag.view(-1, 1, 1), self.L_init, carry.z_L),
+        )
+
+    def re_inittialize_carry(self, reset_flag: torch.Tensor, carry: TinyRecursiveReasoningModel_ACTV1InnerCarry, target_latent: Optional[str] = None):
+        target_device = carry.z_L.device
+        if target_latent == "LH":
+            return TinyRecursiveReasoningModel_ACTV1InnerCarry(
+                z_H=torch.where(
+                    reset_flag.view(-1, 1, 1),
+                    self.init_latent_buffer(self.config.hidden_size, self.forward_dtype, device=target_device),
+                    carry.z_H,
+                ),
+                z_L=torch.where(
+                    reset_flag.view(-1, 1, 1),
+                    self.init_latent_buffer(self.config.hidden_size, self.forward_dtype, device=target_device),
+                    carry.z_L,
+                ),
+            )
+        elif target_latent == "H":
+            return TinyRecursiveReasoningModel_ACTV1InnerCarry(
+                z_H=torch.where(
+                    reset_flag.view(-1, 1, 1),
+                    self.init_latent_buffer(self.config.hidden_size, self.forward_dtype, device=target_device),
+                    carry.z_H,
+                ),
+                z_L=carry.z_L
+            )
+        elif target_latent == "L":
+            return TinyRecursiveReasoningModel_ACTV1InnerCarry(
+                z_H=carry.z_H,
+                z_L=torch.where(
+                    reset_flag.view(-1, 1, 1),
+                    self.init_latent_buffer(self.config.hidden_size, self.forward_dtype, device=target_device),
+                    carry.z_L,
+                )
+            )
+        else:
+            raise ValueError(f"Invalid target_latent value: {target_latent}")
+
+
+    def forward(self, carry: TinyRecursiveReasoningModel_ACTV1InnerCarry, batch: Dict[str, torch.Tensor]) -> Tuple[TinyRecursiveReasoningModel_ACTV1InnerCarry, torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
+        seq_info = dict(
+            cos_sin=self.rotary_emb() if hasattr(self, "rotary_emb") else None,
+        )
+
+        # Input encoding
+        input_embeddings = self._input_embeddings(batch["inputs"], batch["puzzle_identifiers"], batch["group_indices"])
+
+        # Forward iterations
+        it = 0
+        z_H, z_L = carry.z_H, carry.z_L
+        # H_cycles-1 without grad
+        with torch.no_grad():
+            for _H_step in range(self.config.H_cycles-1):
+                for _L_step in range(self.config.L_cycles):
+                    z_L = self.L_level(z_L, z_H + input_embeddings, **seq_info)
+                z_H = self.L_level(z_H, z_L, **seq_info)
+        # 1 with grad
+        for _L_step in range(self.config.L_cycles):
+            z_L = self.L_level(z_L, z_H + input_embeddings, **seq_info)
+        z_H = self.L_level(z_H, z_L, **seq_info)
+
+        # LM Outputs
+        new_carry = TinyRecursiveReasoningModel_ACTV1InnerCarry(z_H=z_H.detach(), z_L=z_L.detach())  # New carry no grad
+        output = self.lm_head(z_H)[:, self.puzzle_emb_len:]
+        q_logits = self.q_head(z_H[:, 0]).to(torch.float32) # Q-head; uses the first puzzle_emb position
+        return new_carry, output, (q_logits[..., 0], q_logits[..., 1])
+
+
+class TinyRecursiveReasoningModel_ACTV1(nn.Module):
+    """ACT wrapper."""
+
+    def __init__(self, config_dict: dict):
+        super().__init__()
+        self.config = TinyRecursiveReasoningModel_ACTV1Config(**config_dict)
+        self.inner = TinyRecursiveReasoningModel_ACTV1_Inner(self.config)
+
+    @property
+    def puzzle_emb(self):
+        return self.inner.puzzle_emb
+
+    @property
+    def group_emb(self):
+        return self.inner.group_emb
+
+    def initial_carry(self, batch: Dict[str, torch.Tensor]):
+        batch_size = batch["inputs"].shape[0]
+
+        return TinyRecursiveReasoningModel_ACTV1Carry(
+            inner_carry=self.inner.empty_carry(batch_size),  # Empty is expected, it will be reseted in first pass as all sequences are halted.
+            
+            steps=torch.zeros((batch_size, ), dtype=torch.int32),
+            halted=torch.ones((batch_size, ), dtype=torch.bool),  # Default to halted
+            
+            current_data={k: torch.empty_like(v) for k, v in batch.items()}
+        )
+
+    def step_inner_carry(self, carry: TinyRecursiveReasoningModel_ACTV1Carry, batch: Dict[str, torch.Tensor]):
+        new_inner_carry = self.inner.reset_carry(carry.halted, carry.inner_carry)
+        return new_inner_carry
+
+    def step_current_data(self, carry: TinyRecursiveReasoningModel_ACTV1Carry, batch: Dict[str, torch.Tensor]):
+        new_current_data = {k: torch.where(carry.halted.view((-1, ) + (1, ) * (batch[k].ndim - 1)), batch[k], v) for k, v in carry.current_data.items()}
+        return new_current_data
+    
+    def step_carry(self, carry: TinyRecursiveReasoningModel_ACTV1Carry, batch: Dict[str, torch.Tensor]):
+        # Step inner carry
+        new_inner_carry = self.step_inner_carry(carry, batch)
+        
+        new_steps = torch.where(carry.halted, 0, carry.steps)
+
+        # Update data, carry (removing halted sequences)        
+        new_current_data = self.step_current_data(carry, batch)
+
+        return new_inner_carry, new_steps, new_current_data
+
+    def forward(self, carry: TinyRecursiveReasoningModel_ACTV1Carry, batch: Dict[str, torch.Tensor]) -> Tuple[TinyRecursiveReasoningModel_ACTV1Carry, Dict[str, torch.Tensor]]:
+
+        new_inner_carry, new_steps, new_current_data = self.step_carry(carry, batch)
+
+        # Forward inner model
+        new_inner_carry, logits, (q_halt_logits, q_continue_logits) = self.inner(new_inner_carry, new_current_data)
+
+        outputs = {
+            "logits": logits,
+            "q_halt_logits": q_halt_logits,
+            "q_continue_logits": q_continue_logits
+        }
+
+        with torch.no_grad():
+            # Step
+            new_steps = new_steps + 1
+            is_last_step = new_steps >= self.config.halt_max_steps
+            
+            halted = is_last_step
+
+            # if training, and ACT is enabled
+            if self.training and (self.config.halt_max_steps > 1) or (self.config.eval_with_ACT):
+
+                # Halt signal
+                # NOTE: During evaluation, always use max steps, this is to guarantee the same halting steps inside a batch for batching purposes
+                
+                if self.config.no_ACT_continue:
+                    halted = halted | (q_halt_logits > self.config.ACT_threshold)
+                else:
+                    halted = halted | (q_halt_logits > q_continue_logits)
+
+                # Exploration
+                min_halt_steps = (torch.rand_like(q_halt_logits) < self.config.halt_exploration_prob) * torch.randint_like(new_steps, low=2, high=self.config.halt_max_steps + 1)
+                halted = halted & (new_steps >= min_halt_steps)
+
+                if not self.config.no_ACT_continue:
+                    # Compute target Q
+                    # NOTE: No replay buffer and target networks for computing target Q-value.
+                    # As batch_size is large, there're many parallel envs.
+                    # Similar concept as PQN https://arxiv.org/abs/2407.04811
+                    _, _, (next_q_halt_logits, next_q_continue_logits), _, _ = self.inner(new_inner_carry, new_current_data)
+                    outputs["target_q_continue"] = torch.sigmoid(torch.where(is_last_step, next_q_halt_logits, torch.maximum(next_q_halt_logits, next_q_continue_logits)))          
+
+        return TinyRecursiveReasoningModel_ACTV1Carry(new_inner_carry, new_steps, halted, new_current_data), outputs
+
+
+class TinyRecursiveReasoningModel_ACTGym(TinyRecursiveReasoningModel_ACTV1):
+    """ACT wrapper."""
+
+    def __init__(self, config_dict: dict):
+        super().__init__(config_dict)
+        self.GymConfig = TinyRecursiveReasoningModel_ACTGymConfig(**config_dict)
+
+    def step_inner_carry(self, carry: TinyRecursiveReasoningModel_ACTV1Carry, batch: Dict[str, torch.Tensor]):
+        # check the first step where all halted and steps is 0
+        new_inner_carry = self.inner.reset_carry(carry.halted & (carry.steps > self.GymConfig.replace_halt_threshold), carry.inner_carry)
+        
+        new_inner_carry = self.inner.re_inittialize_carry(carry.halted & (carry.steps <= self.GymConfig.replace_halt_threshold), new_inner_carry, target_latent=self.GymConfig.target_latent_for_reinit)
+        return new_inner_carry
+
+    def step_carry(self, carry: TinyRecursiveReasoningModel_ACTV1Carry, batch: Dict[str, torch.Tensor]):
+        # Step inner carry
+        new_inner_carry = None
+        if (carry.halted & (carry.steps == 0)).all():
+            new_inner_carry = self.inner.reset_carry(carry.halted, carry.inner_carry)
+        else:
+            new_inner_carry = self.step_inner_carry(carry, batch)
+            carry.halted = carry.halted & (carry.steps > self.GymConfig.replace_halt_threshold)
+
+        new_steps = torch.where(carry.halted, 0, carry.steps)
+
+        # Update data, carry (removing halted sequences)        
+        new_current_data = self.step_current_data(carry, batch)
+
+        return new_inner_carry, new_steps, new_current_data