Auto upload zain 2026-08-12T18:22:01.542466

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+zain/Activation/wandb/run-20260812_175304-hrggki9k/run-hrggki9k.wandb filter=lfs diff=lfs merge=lfs -text
+zain/Activation/wandb/run-20260812_180713-utqzlg67/run-utqzlg67.wandb filter=lfs diff=lfs merge=lfs -text

zain/Activation/README.md

@@ -0,0 +1 @@
+# Activation

zain/Activation/exp.py

@@ -0,0 +1,1265 @@
+"""
+Tiny Llama GLU Lab — Consolidated training library.
+One file: model definition, activation registry, stability monitoring,
+time tracking, dataset builder, and Trainer factory.
+"""
+
+import math
+import os
+import time
+import json
+import re
+from pathlib import Path
+from itertools import chain
+from typing import Dict, Callable, Optional, List, Any, Tuple
+
+import torch
+import torch.nn as nn
+from transformers import (
+    LlamaConfig,
+    LlamaPreTrainedModel,
+    Trainer,
+    TrainerCallback,
+    TrainingArguments,
+    DataCollatorForLanguageModeling,
+    AutoTokenizer,
+    set_seed,
+)
+from transformers.models.llama.modeling_llama import (
+    LlamaAttention,
+    LlamaRMSNorm,
+    LlamaRotaryEmbedding,
+)
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from datasets import load_dataset
+
+
+# =============================================================================
+# 1. ACTIVATION REGISTRY
+# =============================================================================
+
+class GLUActivationRegistry:
+    """Own every gating activation you test. Add new variants in one line."""
+    _registry: Dict[str, Callable[[torch.Tensor], torch.Tensor]] = {}
+
+    @classmethod
+    def register(cls, name: str, fn: Callable[[torch.Tensor], torch.Tensor]) -> None:
+        cls._registry[name] = fn
+
+    @classmethod
+    def get(cls, name: str) -> Callable[[torch.Tensor], torch.Tensor]:
+        if name not in cls._registry:
+            raise KeyError(
+                f"Activation '{name}' not found. Available: {list(cls._registry.keys())}"
+            )
+        return cls._registry[name]
+
+
+# Built-ins
+GLUActivationRegistry.register("silu", nn.functional.silu)
+GLUActivationRegistry.register("swish", nn.functional.silu)
+GLUActivationRegistry.register("relu", nn.functional.relu)
+GLUActivationRegistry.register("gelu", nn.functional.gelu)
+GLUActivationRegistry.register("sigmoid", torch.sigmoid)
+GLUActivationRegistry.register("tanh", torch.tanh)
+GLUActivationRegistry.register("softplus", nn.functional.softplus)
+GLUActivationRegistry.register("linear", lambda x: x)
+
+# Custom
+GLUActivationRegistry.register("s10", lambda x: x * x * torch.sigmoid(x))
+GLUActivationRegistry.register("w1a", lambda x: x * torch.tanh(x))
+
+
+# =============================================================================
+# 2. CONFIG
+# =============================================================================
+
+class TinyLlamaConfig(LlamaConfig):
+    """
+    Exact Llama config plus two fields:
+      - mlp_type: "glu" or "mlp" (standard Transformer MLP)
+      - activation: name of the activation function to use inside the MLP block.
+      - waleed_beta: β_cap for the waleed10 post‑clip (default 10.0)
+    Enforces pure MHA by requiring num_key_value_heads == num_attention_heads.
+    """
+    model_type = "tiny_llama"
+
+    def __init__(
+        self,
+        mlp_type: str = "glu",
+        activation: str = "silu",
+        waleed_beta: float = 10.0,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.mlp_type = mlp_type
+        self.activation = activation
+        self.waleed_beta = waleed_beta
+        if self.num_key_value_heads != self.num_attention_heads:
+            raise ValueError(
+                f"Pure MHA required: num_key_value_heads ({self.num_key_value_heads}) "
+                f"must equal num_attention_heads ({self.num_attention_heads})."
+            )
+
+
+# =============================================================================
+# 3. MODEL
+# =============================================================================
+
+class TinyLlamaMLP(nn.Module):
+    """
+    Unified MLP block supporting:
+      - Standard GLU:   down_proj( act(gate_proj(x)) * up_proj(x) )
+      - Standard MLP:   down_proj( act(up_proj(x)) )
+      - Gated variants (situglu, waleed) with internal tanh scaling.
+      - waleed10 / silu-waleed10 with post‑down‑proj clipping.
+
+    All projections are always named the same way, ensuring consistent
+    tensor logging regardless of activation.
+    """
+    def __init__(self, config: TinyLlamaConfig):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.mlp_type = config.mlp_type
+        self.activation_name = config.activation
+        self.waleed_beta = getattr(config, "waleed_beta", 10.0)
+
+        # Effective intermediate size (scaled for MLP)
+        if self.mlp_type == "glu":
+            effective_intermediate = self.intermediate_size
+        elif self.mlp_type == "mlp":
+            effective_intermediate = int(self.intermediate_size * 1.5)
+            print(f"[MLP] Auto‑scaled intermediate_size from {self.intermediate_size} to {effective_intermediate} for parameter parity.")
+        else:
+            raise ValueError(f"Unknown mlp_type: {self.mlp_type}")
+
+        self.effective_intermediate = effective_intermediate
+
+        # Always define the projections (names consistent across variants)
+        if self.mlp_type == "glu":
+            self.gate_proj = nn.Linear(self.hidden_size, effective_intermediate, bias=False)
+            self.up_proj = nn.Linear(self.hidden_size, effective_intermediate, bias=False)
+        else:  # mlp
+            self.up_proj = nn.Linear(self.hidden_size, effective_intermediate, bias=False)
+
+        self.down_proj = nn.Linear(effective_intermediate, self.hidden_size, bias=False)
+
+        # Activation function (for standard variants)
+        # For gated variants (situglu, waleed) we handle them in forward, but we still need a placeholder.
+        # For waleed10/silu-waleed10 we need linear or silu.
+        if self.mlp_type == "glu":
+            if self.activation_name in ("situglu", "waleed", "situglu_low", "waleedglu_low"):
+                # These will be handled in forward; no act_fn needed.
+                self.act_fn = None
+            elif self.activation_name == "waleed10":
+                self.act_fn = GLUActivationRegistry.get("linear")
+            elif self.activation_name == "silu-waleed10":
+                self.act_fn = GLUActivationRegistry.get("silu")
+            else:
+                self.act_fn = GLUActivationRegistry.get(self.activation_name)
+        else:  # mlp
+            if self.activation_name in ("situglu", "waleed", "situglu_low", "waleedglu_low"):
+                raise ValueError(
+                    f"Activation '{self.activation_name}' requires a gated architecture (GLU). "
+                    f"Please use mlp_type='glu'."
+                )
+            elif self.activation_name == "waleed10":
+                self.act_fn = GLUActivationRegistry.get("linear")
+            elif self.activation_name == "silu-waleed10":
+                self.act_fn = GLUActivationRegistry.get("silu")
+            else:
+                self.act_fn = GLUActivationRegistry.get(self.activation_name)
+
+        # Determine beta values for situglu/waleed variants
+        if self.activation_name in ("situglu_low", "waleedglu_low"):
+            self.beta1 = 2.5
+            self.beta2 = 4.0
+        else:  # original situglu or waleed
+            self.beta1 = 4.0
+            self.beta2 = 25.0
+
+        # Flags for special handling
+        self.is_situglu = self.activation_name in ("situglu", "situglu_low")
+        self.is_waleed = self.activation_name in ("waleed", "waleedglu_low")
+        self.is_waleed10 = self.activation_name in ("waleed10", "silu-waleed10")
+        self.has_sigmoid_gate = self.activation_name.startswith("situglu")  # sigmoid in gate
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if self.mlp_type == "glu":
+            gate = self.gate_proj(x)
+            up = self.up_proj(x)
+
+            if self.is_situglu or self.is_waleed:
+                # Gated variants with tanh scaling
+                if self.has_sigmoid_gate:
+                    gate = self.beta1 * torch.tanh(gate / self.beta1) * torch.sigmoid(gate)
+                else:
+                    gate = self.beta1 * torch.tanh(gate / self.beta1)
+                up = self.beta2 * torch.tanh(up / self.beta2)
+                hidden = gate * up
+            else:
+                # Standard GLU (activation applied to gate)
+                hidden = self.act_fn(gate) * up
+
+            out = self.down_proj(hidden)
+
+        else:  # mlp
+            hidden = self.act_fn(self.up_proj(x))
+            out = self.down_proj(hidden)
+
+        # Post‑clip for waleed10 variants
+        if self.is_waleed10:
+            out = self.waleed_beta * torch.tanh(out / self.waleed_beta)
+
+        return out
+
+
+class TinyLlamaDecoderLayer(nn.Module):
+    def __init__(self, config: TinyLlamaConfig, layer_idx: int):
+        super().__init__()
+        self.hidden_size = config.hidden_size
+        self.self_attn = LlamaAttention(config=config, layer_idx=layer_idx)
+        self.mlp = TinyLlamaMLP(config)
+        self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.post_attention_layernorm = LlamaRMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps
+        )
+        # ---------------------------------------------------------------------
+        # NEW: zero-parameter Identity gateways for residual-stream logging.
+        # These expose the residual tensor as named modules so the hook
+        # registry can capture them with pattern ".*residual.*".
+        # ---------------------------------------------------------------------
+        self.residual_pre_attn = nn.Identity()
+        self.residual_post_attn = nn.Identity()
+        self.residual_post_mlp = nn.Identity()
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
+        **kwargs,
+    ):
+        # --- Attention sub-layer ---
+        residual = hidden_states
+        hidden_states = self.residual_pre_attn(hidden_states)
+        hidden_states = self.input_layernorm(hidden_states)
+        attn_out = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            position_embeddings=position_embeddings,
+        )[0]
+        hidden_states = residual + attn_out
+        hidden_states = self.residual_post_attn(hidden_states)
+
+        # --- MLP sub-layer ---
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+        hidden_states = self.residual_post_mlp(hidden_states)
+        return (hidden_states,)
+
+
+# ----------------------------------------------------------------------------
+# ATTENTION MASK – float mask with 0.0 / -inf (works with all backends)
+# ----------------------------------------------------------------------------
+def _build_causal_mask(
+    attention_mask: Optional[torch.Tensor],
+    seq_len: int,
+    dtype: torch.dtype,
+    device: torch.device,
+) -> torch.Tensor:
+    """
+    Build a 4D float attention mask for scaled_dot_product_attention.
+    - 0.0 where attention is allowed
+    - -inf where it is masked (causal future + padding)
+    """
+    min_value = torch.finfo(dtype).min
+
+    # Causal mask: upper triangle (future) = -inf
+    causal = torch.full((seq_len, seq_len), fill_value=min_value, dtype=dtype, device=device)
+    causal = torch.triu(causal, diagonal=1)
+    causal = causal[None, None, :, :]  # (1, 1, seq_len, seq_len)
+
+    if attention_mask is None:
+        batch_size = 1
+        return causal.expand(batch_size, 1, seq_len, seq_len)
+
+    batch_size = attention_mask.shape[0]
+    causal = causal.expand(batch_size, 1, seq_len, seq_len).clone()
+
+    # Padding: where attention_mask == 0, set to -inf
+    padding = attention_mask[:, None, None, :].to(device) == 0  # (batch, 1, 1, seq_len)
+    causal = causal.masked_fill(padding, min_value)
+
+    return causal
+
+
+# Global flag to print mask message only once
+_MASK_PRINTED = False
+
+
+class TinyLlamaModel(LlamaPreTrainedModel):
+    config_class = TinyLlamaConfig
+
+    def __init__(self, config: TinyLlamaConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+        self.embed_tokens = nn.Embedding(
+            config.vocab_size, config.hidden_size, self.padding_idx
+        )
+        self.layers = nn.ModuleList(
+            [TinyLlamaDecoderLayer(config, i) for i in range(config.num_hidden_layers)]
+        )
+        self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = LlamaRotaryEmbedding(config=config)
+        self.post_init()
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ):
+        global _MASK_PRINTED
+
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        if position_ids is None:
+            seq_len = inputs_embeds.shape[1]
+            position_ids = torch.arange(
+                seq_len, device=inputs_embeds.device
+            ).unsqueeze(0).expand(inputs_embeds.shape[0], -1)
+
+        hidden_states = inputs_embeds
+        position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+        # Build float causal + padding mask (print only once)
+        seq_len = hidden_states.shape[1]
+        causal_mask = _build_causal_mask(
+            attention_mask, seq_len, hidden_states.dtype, hidden_states.device
+        )
+
+        if not _MASK_PRINTED:
+            print("[INFO] Causal mask (float with -inf) applied to all attention layers.")
+            _MASK_PRINTED = True
+
+        for decoder_layer in self.layers:
+            layer_outputs = decoder_layer(
+                hidden_states,
+                attention_mask=causal_mask,
+                position_ids=position_ids,
+                position_embeddings=position_embeddings,
+            )
+            hidden_states = layer_outputs[0]
+
+        hidden_states = self.norm(hidden_states)
+        if not return_dict:
+            return (hidden_states,)
+        return {"last_hidden_state": hidden_states, "hidden_states": None, "attentions": None}
+
+
+class TinyLlamaForCausalLM(LlamaPreTrainedModel):
+    config_class = TinyLlamaConfig
+    _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"}
+
+    def __init__(self, config: TinyLlamaConfig):
+        super().__init__(config)
+        self.model = TinyLlamaModel(config)
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        if config.tie_word_embeddings:
+            self.lm_head.weight = self.model.embed_tokens.weight
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        return_dict: Optional[bool] = None,
+        **kwargs,
+    ):
+        return_dict = (
+            return_dict if return_dict is not None else self.config.use_return_dict
+        )
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            return_dict=return_dict,
+        )
+        hidden_states = outputs["last_hidden_state"] if return_dict else outputs[0]
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(
+                shift_logits.view(-1, self.config.vocab_size), shift_labels.view(-1)
+            )
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=None,
+            hidden_states=None,
+            attentions=None,
+        )
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, **kwargs
+    ):
+        if past_key_values:
+            input_ids = input_ids[:, -1:]
+        position_ids = kwargs.get("position_ids")
+        if attention_mask is not None and position_ids is None:
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -1].unsqueeze(-1)
+        return {
+            "input_ids": input_ids,
+            "position_ids": position_ids,
+            "past_key_values": past_key_values,
+            "attention_mask": attention_mask,
+        }
+
+
+# =============================================================================
+# 4. MONITORING ENGINE
+# =============================================================================
+
+class StatsEngine:
+    """Compute the unified signature for any tensor (now with range + percentiles)."""
+
+    @staticmethod
+    def compute(
+        tensor: torch.Tensor, user_limit: float, dtype_ratio: float
+    ) -> Dict[str, float]:
+        with torch.no_grad():
+            abs_t = tensor.abs()
+            dtype_info = torch.finfo(tensor.dtype)
+            dtype_limit = (
+                dtype_ratio * dtype_info.max
+                if not torch.isinf(torch.tensor(dtype_info.max))
+                else float("inf")
+            )
+
+            # --- Base statistics ---
+            norm = tensor.norm(2).item()
+            mean = tensor.mean().item()
+            std = tensor.std().item()
+            max_abs = abs_t.max().item()
+
+            # --- Exact min / max / range ---
+            t_min = tensor.min().item()
+            t_max = tensor.max().item()
+            t_range = t_max - t_min
+
+            # --- Exact percentiles (float32 CPU for dtype safety) ---
+            p01 = p25 = p50 = p75 = p90 = p95 = p99 = 0.0
+            try:
+                flat_f32 = tensor.detach().reshape(-1).to(torch.float32).cpu()
+                if flat_f32.numel() > 0:
+                    q_vals = torch.quantile(
+                        flat_f32,
+                        torch.tensor(
+                            [0.01, 0.25, 0.5, 0.75, 0.90, 0.95, 0.99],
+                            dtype=torch.float32,
+                        ),
+                    )
+                    p01, p25, p50, p75, p90, p95, p99 = (v.item() for v in q_vals)
+            except Exception:
+                pass  # If quantile fails (very unlikely), leave as 0.0
+
+            return {
+                "norm": norm,
+                "mean": mean,
+                "std": std,
+                "max_abs": max_abs,
+                "frac_near_dtype_limit": (
+                    (abs_t > dtype_limit).float().mean().item()
+                    if not math.isinf(dtype_limit)
+                    else 0.0
+                ),
+                "frac_near_user_limit": (abs_t > user_limit).float().mean().item(),
+                # --- NEW: distributional tail metrics (exact per-tensor) ---
+                "min": t_min,
+                "max": t_max,
+                "range": t_range,
+                "p01": p01,
+                "p25": p25,
+                "p50": p50,
+                "p75": p75,
+                "p90": p90,
+                "p95": p95,
+                "p99": p99,
+            }
+
+
+class StepAccumulator:
+    """
+    Stores per-tensor entries, then aggregates to layer-scope or global-scope
+    using exact population formulas (no tensor retention).
+    """
+
+    def __init__(self):
+        # name -> {numel, norm, mean, std, max_abs, frac_near_dtype_limit, frac_near_user_limit, min, max, range, p01..p99}
+        self.tensors: Dict[str, Dict[str, float]] = {}
+
+    def add(self, name: str, numel: int, stats: Dict[str, float]):
+        new_entry = {"numel": numel, **stats}
+        existing = self.tensors.get(name)
+        self.tensors[name] = (
+            new_entry if existing is None else self._merge_entry(existing, new_entry)
+        )
+
+    @staticmethod
+    def _merge_entry(a: Dict[str, float], b: Dict[str, float]) -> Dict[str, float]:
+        total_n = a["numel"] + b["numel"]
+        if total_n == 0:
+            return a
+        norm = math.sqrt(a["norm"] ** 2 + b["norm"] ** 2)
+        max_abs = max(a["max_abs"], b["max_abs"])
+        mean = (a["mean"] * a["numel"] + b["mean"] * b["numel"]) / total_n
+        ex2 = (
+            a["numel"] * (a["std"] ** 2 + a["mean"] ** 2)
+            + b["numel"] * (b["std"] ** 2 + b["mean"] ** 2)
+        ) / total_n
+        std = math.sqrt(max(0.0, ex2 - mean ** 2))
+        frac_dtype = (
+            a["frac_near_dtype_limit"] * a["numel"] + b["frac_near_dtype_limit"] * b["numel"]
+        ) / total_n
+        frac_user = (
+            a["frac_near_user_limit"] * a["numel"] + b["frac_near_user_limit"] * b["numel"]
+        ) / total_n
+
+        # Exact min/max across micro-batches (gradient accumulation)
+        t_min = min(a.get("min", float("inf")), b.get("min", float("inf")))
+        t_max = max(a.get("max", float("-inf")), b.get("max", float("-inf")))
+        t_range = t_max - t_min
+
+        # Percentiles: weighted average across micro-batches (best effort)
+        def _wpct(key: str) -> float:
+            av = a.get(key, 0.0)
+            bv = b.get(key, 0.0)
+            if av == 0.0 and bv == 0.0:
+                return 0.0
+            return (av * a["numel"] + bv * b["numel"]) / total_n
+
+        return {
+            "numel": total_n,
+            "norm": norm,
+            "mean": mean,
+            "std": std,
+            "max_abs": max_abs,
+            "frac_near_dtype_limit": frac_dtype,
+            "frac_near_user_limit": frac_user,
+            "min": t_min,
+            "max": t_max,
+            "range": t_range,
+            "p01": _wpct("p01"),
+            "p25": _wpct("p25"),
+            "p50": _wpct("p50"),
+            "p75": _wpct("p75"),
+            "p90": _wpct("p90"),
+            "p95": _wpct("p95"),
+            "p99": _wpct("p99"),
+        }
+
+    def clear(self):
+        self.tensors.clear()
+
+    def _aggregate(self, entries: Dict[str, Dict[str, float]]) -> Dict[str, float]:
+        if not entries:
+            return {}
+        numels = [e["numel"] for e in entries.values()]
+        total_n = sum(numels)
+
+        # L2 norm
+        norm = math.sqrt(sum(e["norm"] ** 2 for e in entries.values()))
+        # Max abs
+        max_abs = max(e["max_abs"] for e in entries.values())
+        # Weighted mean
+        mean = sum(e["mean"] * e["numel"] for e in entries.values()) / total_n
+        # Pooled std: sqrt( E[σ² + μ²] - μ_global² )
+        ex2 = (
+            sum(e["numel"] * (e["std"] ** 2 + e["mean"] ** 2) for e in entries.values())
+            / total_n
+        )
+        std = math.sqrt(max(0.0, ex2 - mean ** 2))
+        # Weighted fractions
+        frac_dtype = (
+            sum(e["frac_near_dtype_limit"] * e["numel"] for e in entries.values())
+            / total_n
+        )
+        frac_user = (
+            sum(e["frac_near_user_limit"] * e["numel"] for e in entries.values())
+            / total_n
+        )
+
+        # Exact bounds across all tensors in this scope
+        t_min = min(e.get("min", float("inf")) for e in entries.values())
+        t_max = max(e.get("max", float("-inf")) for e in entries.values())
+        t_range = t_max - t_min
+
+        return {
+            "norm": norm,
+            "mean": mean,
+            "std": std,
+            "max_abs": max_abs,
+            "frac_near_dtype_limit": frac_dtype,
+            "frac_near_user_limit": frac_user,
+            "min": t_min,
+            "max": t_max,
+            "range": t_range,
+            # NOTE: percentiles intentionally omitted from aggregated scopes.
+            # They are only meaningful at per-tensor scope.
+        }
+
+    def get_global_stats(self) -> Dict[str, float]:
+        return self._aggregate(self.tensors)
+
+    def get_layer_stats(self, layer_prefix: str) -> Dict[str, float]:
+        entries = {
+            k: v for k, v in self.tensors.items() if k.startswith(layer_prefix + ".")
+        }
+        return self._aggregate(entries)
+
+
+class HookRegistry:
+    """Attach and throttle forward/backward hooks."""
+
+    def __init__(self, model: nn.Module):
+        self.model = model
+        self.handles: List[torch.utils.hooks.RemovableHandle] = []
+        self.active = False
+
+    def attach_forward(
+        self,
+        module_patterns: List[str],
+        accumulator: StepAccumulator,
+        user_limit: float,
+        dtype_ratio: float,
+    ):
+        for name, module in self.model.named_modules():
+            if any(re.search(p, name) for p in module_patterns):
+                h = module.register_forward_hook(
+                    self._make_forward_hook(name, accumulator, user_limit, dtype_ratio)
+                )
+                self.handles.append(h)
+
+    def attach_backward(
+        self, accumulator: StepAccumulator, user_limit: float, dtype_ratio: float
+    ):
+        for name, param in self.model.named_parameters():
+            if param.requires_grad:
+                h = param.register_hook(
+                    self._make_backward_hook(
+                        f"grad.{name}", accumulator, user_limit, dtype_ratio
+                    )
+                )
+                self.handles.append(h)
+
+    def _make_forward_hook(
+        self, module_name: str, accumulator: StepAccumulator, user_limit: float, dtype_ratio: float
+    ):
+        def hook(module, inp, out):
+            if not self.active:
+                return
+
+            # Modules can return a tensor, a tuple (take first item), or a
+            # dict (e.g. TinyLlamaModel returns {"last_hidden_state": ...}).
+            # Pull out the first real tensor we find; skip cleanly if none.
+            if isinstance(out, dict):
+                out_dict = out
+                out = out_dict.get("last_hidden_state")
+                if out is None:
+                    out = next(
+                        (v for v in out_dict.values() if torch.is_tensor(v)), None
+                    )
+            elif isinstance(out, (tuple, list)):
+                out = out[0] if len(out) > 0 else None
+
+            if not torch.is_tensor(out):
+                return
+
+            stats = StatsEngine.compute(out.detach(), user_limit, dtype_ratio)
+            accumulator.add(f"act.{module_name}", out.numel(), stats)
+
+        return hook
+
+    def _make_backward_hook(
+        self, param_name: str, accumulator: StepAccumulator, user_limit: float, dtype_ratio: float
+    ):
+        def hook(grad):
+            if not self.active:
+                return
+            stats = StatsEngine.compute(grad.detach(), user_limit, dtype_ratio)
+            accumulator.add(param_name, grad.numel(), stats)
+
+        return hook
+
+    def set_active(self, active: bool):
+        self.active = active
+
+    def clear(self):
+        for h in self.handles:
+            h.remove()
+        self.handles.clear()
+
+
+class StabilityMonitorCallback(TrainerCallback):
+    """
+    Full stability instrumentation: grad / param / act statistics
+    at global, per-layer, and per-tensor scope.
+    """
+
+    def __init__(
+        self,
+        model: nn.Module,
+        monitor_every_n_steps: int = 10,
+        module_patterns: Optional[List[str]] = None,
+        user_limits: Optional[Dict[str, float]] = None,
+        dtype_proximity_ratio: float = 0.9,
+        log_scope: Optional[Dict[str, bool]] = None,
+        monitor_during_eval: bool = False,
+    ):
+        self.model = model
+        self.monitor_every_n_steps = monitor_every_n_steps
+        # NEW: default patterns now include residual gateways
+        self.module_patterns = module_patterns or [".*mlp.*", ".*self_attn.*", ".*residual.*"]
+        self.user_limits = user_limits or {"grad": 1.0, "param": 100.0, "act": 50.0}
+        self.dtype_ratio = dtype_proximity_ratio
+        self.log_scope = log_scope or {
+            "global": True,
+            "per_layer": True,
+            "per_tensor": False,
+        }
+        self.monitor_during_eval = monitor_during_eval
+
+        self.accumulator = StepAccumulator()
+        self.hooks = HookRegistry(model)
+        self.hooks.attach_forward(
+            self.module_patterns,
+            self.accumulator,
+            self.user_limits["act"],
+            self.dtype_ratio,
+        )
+        self.hooks.attach_backward(
+            self.accumulator, self.user_limits["grad"], self.dtype_ratio
+        )
+
+        self.pending_metrics: Optional[Dict[str, float]] = None
+
+    def _should_monitor(self, state) -> bool:
+        return state.global_step % self.monitor_every_n_steps == 0
+
+    def on_step_begin(self, args, state, control, **kwargs):
+        if self._should_monitor(state):
+            self.accumulator.clear()
+            self.hooks.set_active(True)
+
+    def on_step_end(self, args, state, control, **kwargs):
+        if not self.hooks.active:
+            return
+
+        # Parameter stats (post-optimizer step)
+        for name, param in self.model.named_parameters():
+            stats = StatsEngine.compute(
+                param.data, self.user_limits["param"], self.dtype_ratio
+            )
+            self.accumulator.add(f"param.{name}", param.numel(), stats)
+
+        self.hooks.set_active(False)
+        self.pending_metrics = self._build_metrics()
+
+    @staticmethod
+    def _kind_of(name: str) -> str:
+        """Classify a tensor key by its source: activation, gradient, or parameter."""
+        if name.startswith("act."):
+            return "act"
+        if name.startswith("grad."):
+            return "grad"
+        if name.startswith("param."):
+            return "param"
+        return "other"
+
+    @staticmethod
+    def _strip_kind(name: str) -> str:
+        if name.startswith("act."):
+            return name[4:]
+        if name.startswith(("grad.", "param.")):
+            return name[5:]
+        return name
+
+    def _build_metrics(self, scope: str = "train") -> Dict[str, float]:
+        metrics: Dict[str, float] = {}
+
+        # --- Global (split by kind: act / grad / param — never pooled together) ---
+        if self.log_scope.get("global", True):
+            by_kind: Dict[str, Dict[str, Dict[str, float]]] = {}
+            for k, v in self.accumulator.tensors.items():
+                by_kind.setdefault(self._kind_of(k), {})[k] = v
+
+            for kind, entries in by_kind.items():
+                stats = self.accumulator._aggregate(entries)
+                for kk, vv in stats.items():
+                    metrics[f"{scope}/global/{kind}/{kk}"] = vv
+
+        # --- Per-layer (group by model.layers.{i}, split by kind) ---
+        if self.log_scope.get("per_layer", True):
+            layer_prefixes = set()
+            for name in self.accumulator.tensors:
+                clean = self._strip_kind(name)
+                parts = clean.split(".")
+                for i, p in enumerate(parts):
+                    if p == "layers" and i + 1 < len(parts):
+                        prefix = ".".join(parts[: i + 2])
+                        layer_prefixes.add(prefix)
+
+            for prefix in layer_prefixes:
+                by_kind: Dict[str, Dict[str, Dict[str, float]]] = {}
+                for k, v in self.accumulator.tensors.items():
+                    clean = self._strip_kind(k)
+                    if clean.startswith(prefix + ".") or clean == prefix:
+                        by_kind.setdefault(self._kind_of(k), {})[k] = v
+
+                safe = prefix.replace(".", "_")
+                for kind, entries in by_kind.items():
+                    if not entries:
+                        continue
+                    stats = self.accumulator._aggregate(entries)
+                    for kk, vv in stats.items():
+                        metrics[f"{scope}/layer_{safe}/{kind}/{kk}"] = vv
+
+        # --- Per-tensor (exact percentiles live here) ---
+        if self.log_scope.get("per_tensor", False):
+            for name, stats in self.accumulator.tensors.items():
+                safe = name.replace(".", "_")
+                for kk, vv in stats.items():
+                    if kk == "numel":
+                        continue
+                    metrics[f"{scope}/tensor_{safe}/{kk}"] = vv
+
+        return metrics
+
+    def on_log(self, args, state, control, logs=None, **kwargs):
+        if logs is not None and self.pending_metrics is not None:
+            logs.update(self.pending_metrics)
+            self.pending_metrics = None
+
+    def on_prediction_step(self, args, state, control, **kwargs):
+        """Fires once per eval/predict batch. Trainer.evaluate() calls this
+        for every batch in the eval loop, then calls self.log(output.metrics)
+        (which dispatches on_log to every callback, including the wandb/
+        tensorboard reporting callbacks) BEFORE on_evaluate() runs. So to get
+        eval-time stats into that same on_log dispatch, we have to build
+        pending_metrics here, not in on_evaluate — by the time on_evaluate
+        fires, self.log() has already happened and it's too late.
+        """
+        if not self.monitor_during_eval:
+            return
+        if not self.hooks.active:
+            # First batch of this eval pass: start a fresh accumulation and
+            # snapshot parameter stats once (they don't change during eval).
+            self.accumulator.clear()
+            self.hooks.set_active(True)
+            for name, param in self.model.named_parameters():
+                stats = StatsEngine.compute(
+                    param.data, self.user_limits["param"], self.dtype_ratio
+                )
+                self.accumulator.add(f"param.{name}", param.numel(), stats)
+        self.pending_metrics = self._build_metrics(scope="eval")
+
+    def on_evaluate(self, args, state, control, metrics=None, **kwargs):
+        self.hooks.set_active(False)
+        self.accumulator.clear()
+
+
+class TimeTrackerCallback(TrainerCallback):
+    """Precise training & eval timing with remaining-time estimates."""
+
+    def __init__(self):
+        self.step_start: Optional[float] = None
+        self.epoch_start: Optional[float] = None
+        self.total_train_time = 0.0
+        self.step_times: List[float] = []
+
+    def on_epoch_begin(self, args, state, control, **kwargs):
+        self.epoch_start = time.perf_counter()
+
+    def on_step_begin(self, args, state, control, **kwargs):
+        self.step_start = time.perf_counter()
+
+    def on_step_end(self, args, state, control, **kwargs):
+        if self.step_start is not None:
+            dt = time.perf_counter() - self.step_start
+            self.step_times.append(dt)
+            self.total_train_time += dt
+            self.step_start = None
+
+    def on_log(self, args, state, control, logs=None, **kwargs):
+        if logs is None:
+            return
+
+        logs["train/total_time_seconds"] = self.total_train_time
+
+        if self.step_times:
+            recent = self.step_times[-100:]
+            logs["train/time_per_step_avg"] = sum(recent) / len(recent)
+
+        if self.epoch_start is not None:
+            logs["train/epoch_time_elapsed"] = time.perf_counter() - self.epoch_start
+
+        if state.max_steps and state.global_step > 0:
+            avg = self.total_train_time / state.global_step
+            remaining = (state.max_steps - state.global_step) * avg
+            logs["train/estimated_remaining_minutes"] = remaining / 60.0
+
+    def on_evaluate(self, args, state, control, metrics=None, **kwargs):
+        pass
+
+
+class MetricsLoggerCallback(TrainerCallback):
+    """Persist every logged dict as JSONL in the output dir."""
+
+    def __init__(self, output_dir: str):
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self.log_file = self.output_dir / "training_log.jsonl"
+
+    def on_log(self, args, state, control, logs=None, **kwargs):
+        if logs is None:
+            return
+        entry = {
+            "step": state.global_step,
+            "epoch": state.epoch,
+            "timestamp": time.time(),
+            **logs,
+        }
+        with open(self.log_file, "a") as f:
+            f.write(json.dumps(entry, default=str) + "\n")
+
+
+# =============================================================================
+# NEW: CONTAMINATION CALLBACK
+# =============================================================================
+
+class ContaminationCallback(TrainerCallback):
+    """
+    Intentionally corrupt input_ids and labels for a window of steps,
+    causing a controlled loss spike. Supports two modes:
+      - "shift": every token ID is incremented by 1 (mod vocab_size)
+      - "random": token IDs are replaced with uniform random IDs
+    Fraction controls what proportion of tokens (per sequence) are corrupted.
+    """
+    def __init__(
+        self,
+        vocab_size: int,
+        enabled: bool = False,
+        start_step: int = 0,
+        duration_steps: int = 0,
+        mode: str = "shift",
+        fraction: float = 1.0,
+        seed: Optional[int] = None,
+    ):
+        self.vocab_size = vocab_size
+        self.enabled = enabled
+        self.start_step = start_step
+        self.duration_steps = duration_steps
+        self.mode = mode
+        self.fraction = fraction
+        self.seed = seed
+
+        # Create a dedicated generator for reproducibility
+        self.generator = torch.Generator()
+        if seed is not None:
+            self.generator.manual_seed(seed)
+
+        self._active = False
+
+    def _should_corrupt(self, state) -> bool:
+        if not self.enabled:
+            return False
+        step = state.global_step
+        return self.start_step <= step < self.start_step + self.duration_steps
+
+    def on_step_begin(self, args, state, control, **kwargs):
+        if not self._should_corrupt(state):
+            return
+
+        # The batch is passed in kwargs under key "inputs" (Trainer convention).
+        # We also need to grab the model's device to generate tensors on the same device.
+        batch = kwargs.get("inputs")
+        if batch is None:
+            return
+
+        # We need the device – get it from the model or from the input tensors.
+        # We can access the model through the trainer? Not directly here.
+        # But we can infer device from batch tensors.
+        if not isinstance(batch, dict):
+            return
+
+        input_ids = batch.get("input_ids")
+        labels = batch.get("labels")
+        attention_mask = batch.get("attention_mask")
+
+        if input_ids is None or labels is None:
+            return
+
+        device = input_ids.device
+        batch_size, seq_len = input_ids.shape
+
+        # Build mask of positions to corrupt (based on fraction)
+        # We generate a mask of shape (batch_size, seq_len) with True for positions to corrupt.
+        if self.fraction >= 1.0:
+            corrupt_mask = torch.ones((batch_size, seq_len), dtype=torch.bool, device=device)
+        elif self.fraction <= 0.0:
+            return  # nothing to corrupt
+        else:
+            # Generate random floats and threshold
+            rand = torch.rand((batch_size, seq_len), generator=self.generator, device=device)
+            corrupt_mask = rand < self.fraction
+
+        # If attention_mask is present, exclude padding positions from corruption
+        if attention_mask is not None:
+            # attention_mask is 1 for real tokens, 0 for padding
+            padding_mask = attention_mask == 0
+            corrupt_mask = corrupt_mask & (~padding_mask)
+
+        if not corrupt_mask.any():
+            return
+
+        # Apply corruption
+        if self.mode == "shift":
+            # Increment by 1 modulo vocab_size
+            # We need to handle vocab_size properly (IDs go 0..vocab_size-1)
+            input_ids_corrupt = (input_ids + 1) % self.vocab_size
+            labels_corrupt = (labels + 1) % self.vocab_size
+        elif self.mode == "random":
+            # Replace with uniform random IDs
+            # Create random tensor of same shape
+            random_ids = torch.randint(
+                0, self.vocab_size, input_ids.shape,
+                generator=self.generator, device=device
+            )
+            input_ids_corrupt = random_ids
+            labels_corrupt = random_ids
+        else:
+            raise ValueError(f"Unknown contamination mode: {self.mode}")
+
+        # Apply only where mask is True
+        input_ids.masked_scatter_(corrupt_mask, input_ids_corrupt[corrupt_mask])
+        labels.masked_scatter_(corrupt_mask, labels_corrupt[corrupt_mask])
+
+        # Reassign into batch (mutated in-place)
+        batch["input_ids"] = input_ids
+        batch["labels"] = labels
+
+        # Log a one‑time message when contamination starts
+        if not self._active:
+            print(f"[Contamination] Started at step {state.global_step} for {self.duration_steps} steps (mode={self.mode})")
+            self._active = True
+
+    def on_step_end(self, args, state, control, **kwargs):
+        # If we just passed the end of the window, de‑activate
+        if self._active and state.global_step >= self.start_step + self.duration_steps:
+            print(f"[Contamination] Ended at step {state.global_step}")
+            self._active = False
+
+
+# =============================================================================
+# 5. DATA & TRAINER FACTORY
+# =============================================================================
+
+def build_dataset(
+    tokenizer,
+    max_seq_len: int = 512,
+    split: str = "train",
+    dataset_name: str = "roneneldan/TinyStories",
+    max_samples: Optional[int] = None,
+):
+    """Concatenate and chunk TinyStories for causal LM. Fast path with multiprocessing."""
+    ds = load_dataset(dataset_name, split=split)
+    
+    if max_samples is not None and split == "train":
+        ds = ds.select(range(min(max_samples, len(ds))))
+        print(f"[Dataset] Using first {len(ds)} samples for training (max_samples={max_samples})")
+
+    def tokenize(examples):
+        out = tokenizer(examples["text"], add_special_tokens=False)
+        eos_id = tokenizer.eos_token_id
+        out["input_ids"] = [ids + [eos_id] for ids in out["input_ids"]]
+        if "attention_mask" in out:
+            out["attention_mask"] = [mask + [1] for mask in out["attention_mask"]]
+        return out
+
+    tokenized = ds.map(
+        tokenize,
+        batched=True,
+        num_proc=4,
+        remove_columns=ds.column_names,
+        desc=f"Tokenizing {split}",
+    )
+
+    def group_texts(examples):
+        concatenated = {
+            k: list(chain.from_iterable(examples[k])) for k in examples.keys()
+        }
+        total_length = len(concatenated[list(examples.keys())[0]])
+        total_length = (total_length // max_seq_len) * max_seq_len
+        result = {
+            k: [t[i : i + max_seq_len] for i in range(0, total_length, max_seq_len)]
+            for k, t in concatenated.items()
+        }
+        result["labels"] = result["input_ids"].copy()
+        return result
+
+    return tokenized.map(
+        group_texts,
+        batched=True,
+        batch_size=10000,
+        num_proc=4,
+        desc=f"Chunking {split}",
+    )
+
+
+def create_trainer(
+    model,
+    tokenizer,
+    config: Dict[str, Any],
+    train_dataset,
+    eval_dataset=None,
+):
+    """Assemble HF Trainer with all custom callbacks."""
+    tc = config.get("training", {})
+    mc = config.get("monitor", {})
+
+    run_name = tc.get("run_name", None)
+
+    args = TrainingArguments(
+        output_dir=tc.get("output_dir", "./out"),
+        run_name=run_name,
+        num_train_epochs=tc.get("num_train_epochs", 3),
+        per_device_train_batch_size=tc.get("per_device_train_batch_size", 16),
+        per_device_eval_batch_size=tc.get("per_device_eval_batch_size", 16),
+        gradient_accumulation_steps=tc.get("gradient_accumulation_steps", 4),
+        learning_rate=tc.get("learning_rate", 3e-4),
+        weight_decay=tc.get("weight_decay", 0.0),
+        max_grad_norm=tc.get("max_grad_norm", 1.0),
+        optim=tc.get("optim", "adamw_torch"),
+        warmup_steps=tc.get("warmup_steps", 0),
+        lr_scheduler_type=tc.get("lr_scheduler_type", "cosine"),
+        bf16=tc.get("bf16", True),
+        logging_steps=tc.get("logging_steps", 10),
+        eval_strategy=tc.get("eval_strategy", "steps"),
+        eval_steps=tc.get("eval_steps", 500),
+        save_strategy=tc.get("save_strategy", "steps"),
+        save_steps=tc.get("save_steps", 1000),
+        load_best_model_at_end=tc.get("load_best_model_at_end", False),
+        report_to=tc.get("report_to", "tensorboard"),
+        push_to_hub=tc.get("push_to_hub", False),
+        hub_model_id=tc.get("hub_model_id", None),
+        hub_token=tc.get("hub_token") or os.environ.get("HF_TOKEN"),
+        max_steps=tc.get("max_steps", -1),
+        seed=tc.get("seed", 42),
+        data_seed=tc.get("data_seed", 42),
+        remove_unused_columns=False,
+    )
+
+    callbacks = [
+        TimeTrackerCallback(),
+    ]
+
+    # --- Contamination callback (optional) ---
+    cc = config.get("contamination", {})
+    if cc.get("enabled", False):
+        vocab_size = model.config.vocab_size
+        callbacks.append(
+            ContaminationCallback(
+                vocab_size=vocab_size,
+                enabled=True,
+                start_step=cc.get("start_step", 0),
+                duration_steps=cc.get("duration_steps", 0),
+                mode=cc.get("mode", "shift"),
+                fraction=cc.get("fraction", 1.0),
+                seed=cc.get("seed", None),
+            )
+        )
+
+    if mc.get("enabled", True):
+        callbacks.append(
+            StabilityMonitorCallback(
+                model=model,
+                monitor_every_n_steps=mc.get("monitor_every_n_steps", 10),
+                module_patterns=mc.get("module_patterns", [".*mlp.*", ".*self_attn.*", ".*residual.*"]),
+                user_limits=mc.get(
+                    "user_limits", {"grad": 1.0, "param": 100.0, "act": 50.0}
+                ),
+                dtype_proximity_ratio=mc.get("dtype_proximity_ratio", 0.9),
+                log_scope=mc.get(
+                    "log_scope",
+                    {"global": True, "per_layer": True, "per_tensor": False},
+                ),
+                monitor_during_eval=mc.get("monitor_during_eval", False),
+            )
+        )
+
+    callbacks.append(MetricsLoggerCallback(args.output_dir))
+
+    collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+
+    trainer = Trainer(
+        model=model,
+        args=args,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset,
+        data_collator=collator,
+        callbacks=callbacks,
+    )
+
+    # Move reporting integrations to the end
+    try:
+        from transformers.integrations import get_reporting_integration_callbacks
+
+        reporting_types = tuple(get_reporting_integration_callbacks(args.report_to))
+    except Exception:
+        reporting_types = ()
+
+    if reporting_types:
+        handler = trainer.callback_handler
+        reporting_cbs = [cb for cb in handler.callbacks if isinstance(cb, reporting_types)]
+        other_cbs = [cb for cb in handler.callbacks if not isinstance(cb, reporting_types)]
+        handler.callbacks = other_cbs + reporting_cbs
+
+    return trainer

zain/Activation/llm_analyzer_wandb.py

@@ -0,0 +1,570 @@
+"""
+Multi-LLM Activation & Loss Analyzer with WandB Logging
+
+Evaluates multiple language models on WikiText dataset,
+computing per-tensor and global activation statistics (mean, max_abs, std, norm)
+and logging to Weights & Biases with separate runs per model.
+
+Requirements:
+    pip install transformers torch datasets wandb tqdm
+
+Usage:
+    export WANDB_PROJECT="llm-activation-analysis"
+    export WANDB_API_KEY="your-key"
+    python llm_analyzer_wandb.py
+"""
+
+import math
+import torch
+import torch.nn.functional as F
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig
+from datasets import load_dataset
+from typing import List, Dict, Optional, Union, Tuple
+from dataclasses import dataclass, asdict
+from collections import defaultdict
+import json
+import warnings
+import os
+from tqdm import tqdm
+
+import wandb
+
+warnings.filterwarnings("ignore")
+
+
+@dataclass
+class TensorStats:
+    mean: float
+    max_abs: float
+    std: float
+    norm: float
+    numel: int
+
+
+@dataclass
+class ModelResult:
+    model_name: str
+    loss: float
+    perplexity: float
+    global_act: TensorStats
+    layer_acts: Dict[str, TensorStats]
+    num_tokens: int
+    num_layers: int
+    hidden_size: int
+    num_params: int
+
+
+class ActivationHookManager:
+    """Manages forward hooks to capture activations from every tensor."""
+
+    def __init__(self):
+        self.activations = {}
+        self.hooks = []
+        # Set once per batch via set_attention_mask(); used to exclude
+        # padding-token positions from activation statistics.
+        self._attention_mask: Optional[torch.Tensor] = None
+
+    def set_attention_mask(self, attention_mask: Optional[torch.Tensor]):
+        """Call once per batch before the forward pass so hooks can mask
+        out padding positions when computing stats."""
+        self._attention_mask = (
+            attention_mask.detach().cpu() if attention_mask is not None else None
+        )
+
+    def _make_hook(self, name: str):
+        def hook(module, input, output):
+            # Handle different output types
+            if isinstance(output, torch.Tensor):
+                tensor = output
+            elif isinstance(output, tuple) and isinstance(output[0], torch.Tensor):
+                tensor = output[0]
+            else:
+                return
+
+            # Detach and move to CPU to avoid GPU memory blowup
+            self.activations[name] = tensor.detach().cpu().float()
+        return hook
+
+    def register_hooks(self, model: torch.nn.Module):
+        """Register hooks on all modules that produce activations."""
+        for name, module in model.named_modules():
+            # Skip trivial containers
+            if len(list(module.children())) == 0 and hasattr(module, 'forward'):
+                hook = module.register_forward_hook(self._make_hook(name))
+                self.hooks.append(hook)
+
+    def clear(self):
+        self.activations.clear()
+
+    def remove_hooks(self):
+        for hook in self.hooks:
+            hook.remove()
+        self.hooks.clear()
+
+    def _select_real_tokens(self, tensor: torch.Tensor) -> torch.Tensor:
+        """
+        BUG FIX: previously every captured tensor (including activations at
+        padding-token positions) was flattened and used as-is. With
+        padding_side="left" and a small batch_size, the padding fraction
+        varies a lot batch-to-batch, so padding-token activations (which
+        are real, non-zero values — not zeros) were silently mixed into
+        mean/std/max_abs/norm, biasing exactly the saturation signal this
+        script exists to measure.
+
+        Here we mask out padding positions whenever a tensor's shape is
+        consistent with (batch, seq_len, ...) against the stored
+        attention_mask (batch, seq_len). Tensors that don't match that
+        shape (e.g. a module operating on the pooled/final dimension only)
+        are left as-is rather than guessing.
+        """
+        mask = self._attention_mask
+        if mask is None or tensor.dim() < 2:
+            return tensor.reshape(-1)
+        if tensor.shape[0] != mask.shape[0] or tensor.shape[1] != mask.shape[1]:
+            return tensor.reshape(-1)
+
+        bool_mask = mask.bool()
+        # Expand mask across any trailing dims (e.g. hidden_size) and select.
+        expand_shape = bool_mask.shape + (1,) * (tensor.dim() - 2)
+        bool_mask = bool_mask.view(expand_shape).expand_as(tensor)
+        return tensor[bool_mask].reshape(-1)
+
+    def compute_stats(self) -> Dict[str, TensorStats]:
+        """Compute statistics for all captured activations, excluding
+        padding-token positions where identifiable."""
+        stats = {}
+        for name, tensor in self.activations.items():
+            if tensor.numel() == 0:
+                continue
+            flat = self._select_real_tokens(tensor)
+            if flat.numel() == 0:
+                continue
+            stats[name] = TensorStats(
+                mean=flat.mean().item(),
+                max_abs=flat.abs().max().item(),
+                std=flat.std().item(),
+                norm=flat.norm().item(),
+                numel=flat.numel()
+            )
+        return stats
+
+
+class LLMAnalyzer:
+    def __init__(
+        self,
+        device: Optional[str] = None,
+        max_length: int = 512,
+        max_samples: int = 1000,  # number of wikitext samples to eval
+        batch_size: int = 4,
+        dtype: torch.dtype = torch.float16,
+        wandb_project: Optional[str] = None,
+    ):
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+        self.max_length = max_length
+        self.max_samples = max_samples
+        self.batch_size = batch_size
+        self.dtype = dtype if self.device == "cuda" else torch.float32
+        self.wandb_project = wandb_project or os.environ.get("WANDB_PROJECT", "llm-activation-analysis")
+        self._cache = {}
+
+    def load_dataset(self, split: str = "test"):
+        """Load Salesforce/wikitext dataset."""
+        print(f"[Dataset] Loading Salesforce/wikitext ({split}) ...")
+        ds = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split=split)
+        # Filter out empty lines
+        texts = [t for t in ds["text"] if len(t.strip()) > 50]
+        print(f"[Dataset] Loaded {len(texts)} non-empty samples")
+        return texts[:self.max_samples]
+
+    def load_model(self, model_name: str):
+        """Load model and tokenizer with caching."""
+        if model_name in self._cache:
+            return self._cache[model_name]
+
+        print(f"[Loading] {model_name} ...")
+
+        tokenizer = AutoTokenizer.from_pretrained(
+            model_name,
+            trust_remote_code=True,
+            padding_side="left"
+        )
+        if tokenizer.pad_token is None:
+            tokenizer.pad_token = tokenizer.eos_token
+
+        config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
+
+        model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            config=config,
+            torch_dtype=self.dtype,
+            device_map="auto" if self.device == "cuda" else None,
+            trust_remote_code=True,
+        )
+
+        if self.device == "cpu":
+            model = model.to(self.device)
+
+        model.eval()
+
+        num_params = sum(p.numel() for p in model.parameters())
+
+        self._cache[model_name] = (tokenizer, model, config, num_params)
+        print(f"[Loaded]  {model_name} | Params: {num_params/1e6:.1f}M | Layers: {config.num_hidden_layers} | Hidden: {config.hidden_size}")
+        return tokenizer, model, config, num_params
+
+    def compute(
+        self,
+        model_names: List[str],
+    ) -> List[ModelResult]:
+        """
+        Compute loss and per-tensor activation statistics for multiple models.
+        Logs each model as a separate WandB run.
+        """
+        texts = self.load_dataset()
+        results = []
+
+        for model_name in model_names:
+            try:
+                result = self._evaluate_model(model_name, texts)
+                results.append(result)
+            except Exception as e:
+                print(f"[Error] {model_name}: {e}")
+                import traceback
+                traceback.print_exc()
+                continue
+
+        return results
+
+    def _evaluate_model(
+        self,
+        model_name: str,
+        texts: List[str],
+    ) -> ModelResult:
+        tokenizer, model, config, num_params = self.load_model(model_name)
+
+        # Initialize WandB run for this model
+        run_name = model_name.replace("/", "-")
+        wandb.init(
+            project=self.wandb_project,
+            name=run_name,
+            config={
+                "model": model_name,
+                "max_length": self.max_length,
+                "max_samples": self.max_samples,
+                "batch_size": self.batch_size,
+                "dtype": str(self.dtype),
+                "num_params": num_params,
+                "num_layers": config.num_hidden_layers,
+                "hidden_size": config.hidden_size,
+            },
+            reinit=True
+        )
+
+        hook_mgr = ActivationHookManager()
+        hook_mgr.register_hooks(model)
+
+        total_loss = 0.0
+        total_tokens = 0
+
+        # Global activation accumulator
+        global_acts = []
+
+        # Per-layer activation accumulators
+        # We'll aggregate stats across batches, then compute final stats
+        layer_act_values = defaultdict(list)
+
+        num_batches = (len(texts) + self.batch_size - 1) // self.batch_size
+
+        for i in tqdm(range(0, len(texts), self.batch_size), desc=f"Eval {run_name}", total=num_batches):
+            batch_texts = texts[i:i + self.batch_size]
+
+            # Tokenize
+            inputs = tokenizer(
+                batch_texts,
+                return_tensors="pt",
+                truncation=True,
+                max_length=self.max_length,
+                padding=True
+            )
+
+            # Move to device
+            if self.device == "cuda" and hasattr(model, "device"):
+                # model is on auto device map
+                input_ids = inputs["input_ids"]
+                if hasattr(model, "device") and model.device != torch.device("meta"):
+                    input_ids = input_ids.to(model.device)
+                attention_mask = inputs.get("attention_mask")
+                if attention_mask is not None:
+                    attention_mask = attention_mask.to(input_ids.device)
+            else:
+                input_ids = inputs["input_ids"].to(self.device)
+                attention_mask = inputs.get("attention_mask")
+                if attention_mask is not None:
+                    attention_mask = attention_mask.to(self.device)
+
+            labels = input_ids.clone()
+
+            # BUG FIX: with padding_side="left" and no explicit position_ids,
+            # HF's default `position_ids = arange(seq_len)` is applied
+            # identically to every row in the batch, regardless of how much
+            # left-padding precedes the real tokens in that row (verified
+            # against transformers' LlamaModel.forward / GPT2Model.forward
+            # source — neither adjusts for padding when position_ids=None).
+            # That means a real token's absolute position (and therefore its
+            # RoPE rotation / absolute position embedding) depends on how
+            # much padding happened to precede it in this particular batch,
+            # not on its logical position within its own sequence. This
+            # silently corrupts logits -> loss -> perplexity, with the
+            # amount of corruption varying batch-to-batch. Fix: derive
+            # position_ids from attention_mask so they restart at 0 for the
+            # first real token of every row, and are stable (0) on padding.
+            if attention_mask is not None:
+                position_ids = attention_mask.long().cumsum(-1) - 1
+                position_ids.masked_fill_(attention_mask == 0, 0)
+            else:
+                position_ids = None
+
+            hook_mgr.set_attention_mask(attention_mask)
+
+            with torch.no_grad():
+                outputs = model(
+                    input_ids=input_ids,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    labels=labels,
+                )
+
+            # --- Loss Computation ---
+            logits = outputs.logits
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            shift_mask = attention_mask[..., 1:].contiguous() if attention_mask is not None else None
+
+            loss_fct = torch.nn.CrossEntropyLoss(reduction="none")
+            token_losses = loss_fct(
+                shift_logits.view(-1, shift_logits.size(-1)),
+                shift_labels.view(-1)
+            )
+
+            if shift_mask is not None:
+                token_losses = token_losses * shift_mask.view(-1)
+                num_valid_tokens = shift_mask.sum().item()
+            else:
+                num_valid_tokens = token_losses.numel()
+
+            batch_loss = token_losses.sum().item()
+            total_loss += batch_loss
+            total_tokens += num_valid_tokens
+
+            # --- Activation Statistics ---
+            # Get activations captured by hooks
+            act_stats = hook_mgr.compute_stats()
+
+            for name, stats in act_stats.items():
+                # Skip dtype-limit fraction metrics entirely
+                # We only log mean, max_abs, std, norm
+
+                # For global: aggregate raw values
+                # We can't store all raw values due to memory, so we store running sums
+                # But for accurate std across all batches, we need a streaming algorithm
+                # For simplicity and correctness, we'll store per-batch stats and weight them
+                layer_act_values[name].append(asdict(stats))
+
+            hook_mgr.clear()
+
+            # Log per-batch metrics to wandb
+            if num_valid_tokens > 0:
+                batch_avg_loss = batch_loss / num_valid_tokens
+                wandb.log({
+                    "batch_loss": batch_avg_loss,
+                    "batch_perplexity": torch.exp(torch.tensor(batch_avg_loss)).item(),
+                    "batch_tokens": num_valid_tokens,
+                    "progress": i / len(texts)
+                }, step=i)
+
+        hook_mgr.remove_hooks()
+
+        # --- Final Aggregation ---
+        avg_loss = total_loss / max(total_tokens, 1)
+        perplexity = torch.exp(torch.tensor(avg_loss)).item()
+
+        # Aggregate per-tensor stats across all batches
+        # Weighted by numel for mean, max for max_abs, pooled std, pooled norm
+        final_layer_stats = {}
+
+        for name, batch_stats_list in layer_act_values.items():
+            total_numel = sum(s["numel"] for s in batch_stats_list)
+            if total_numel == 0:
+                continue
+
+            # Weighted mean
+            weighted_mean = sum(s["mean"] * s["numel"] for s in batch_stats_list) / total_numel
+
+            # Max abs across all batches
+            max_abs = max(s["max_abs"] for s in batch_stats_list)
+
+            # BUG FIX: the previous formula (weighted average of per-batch
+            # variances only) drops the between-batch term that accounts for
+            # per-batch means differing from the global mean. Whenever batch
+            # means differ (they will — different texts, different lengths),
+            # this systematically UNDERESTIMATES the true global std — in a
+            # quick numeric test with two batches of different means this was
+            # off by ~2.8x. Correct pooled-variance formula (population form,
+            # matches exp.py's StepAccumulator._merge_entry):
+            #   E[X^2] = weighted_avg(var_i + mean_i^2)
+            #   Var(X) = E[X^2] - mean_global^2
+            ex2 = sum(
+                s["numel"] * (s["std"] ** 2 + s["mean"] ** 2) for s in batch_stats_list
+            ) / total_numel
+            pooled_std = math.sqrt(max(0.0, ex2 - weighted_mean ** 2))
+
+            # Norm: sqrt(sum of squared norms / total_numel) * sqrt(total_numel)
+            # Actually norm^2 = sum(x_i^2), so pooled_norm = sqrt(sum(norm_i^2))
+            pooled_norm = (sum(s["norm"] ** 2 for s in batch_stats_list)) ** 0.5
+
+            final_layer_stats[name] = TensorStats(
+                mean=weighted_mean,
+                max_abs=max_abs,
+                std=pooled_std,
+                norm=pooled_norm,
+                numel=total_numel
+            )
+
+        # Compute global stats across all layers
+        if final_layer_stats:
+            all_numel = sum(s.numel for s in final_layer_stats.values())
+            global_mean = sum(s.mean * s.numel for s in final_layer_stats.values()) / all_numel
+            global_max_abs = max(s.max_abs for s in final_layer_stats.values())
+            # Same pooled-std correction as above — same bug was present here.
+            global_ex2 = sum(
+                s.numel * (s.std ** 2 + s.mean ** 2) for s in final_layer_stats.values()
+            ) / all_numel
+            global_std = math.sqrt(max(0.0, global_ex2 - global_mean ** 2))
+            global_norm = (sum(s.norm ** 2 for s in final_layer_stats.values())) ** 0.5
+
+            global_stats = TensorStats(
+                mean=global_mean,
+                max_abs=global_max_abs,
+                std=global_std,
+                norm=global_norm,
+                numel=all_numel
+            )
+        else:
+            global_stats = TensorStats(0.0, 0.0, 0.0, 0.0, 0)
+
+        result = ModelResult(
+            model_name=model_name,
+            loss=avg_loss,
+            perplexity=perplexity,
+            global_act=global_stats,
+            layer_acts=final_layer_stats,
+            num_tokens=total_tokens,
+            num_layers=config.num_hidden_layers,
+            hidden_size=config.hidden_size,
+            num_params=num_params
+        )
+
+        # --- WandB Logging ---
+        self._log_to_wandb(result)
+        wandb.finish()
+
+        return result
+
+    def _log_to_wandb(self, result: ModelResult):
+        """Log final metrics to WandB. No frac_near_dtype_limit."""
+
+        # Global metrics
+        wandb.log({
+            "final/loss": result.loss,
+            "final/perplexity": result.perplexity,
+            "final/num_tokens": result.num_tokens,
+
+            "train/global/act/mean": result.global_act.mean,
+            "train/global/act/max_abs": result.global_act.max_abs,
+            "train/global/act/std": result.global_act.std,
+            "train/global/act/norm": result.global_act.norm,
+            # Intentionally NOT logging frac_near_dtype_limit or frac_near_user_limit
+        })
+
+        # Per-tensor (per-layer) metrics
+        # Organize by layer for cleaner WandB UI
+        for tensor_name, stats in result.layer_acts.items():
+            # Clean name for wandb: replace dots with slashes
+            clean_name = tensor_name.replace(".", "/")
+
+            wandb.log({
+                f"train/{clean_name}/act/mean": stats.mean,
+                f"train/{clean_name}/act/max_abs": stats.max_abs,
+                f"train/{clean_name}/act/std": stats.std,
+                f"train/{clean_name}/act/norm": stats.norm,
+                # No frac_near_dtype_limit
+            })
+
+        # Also log as a wandb.Table for easy comparison
+        table_data = []
+        for tensor_name, stats in sorted(result.layer_acts.items()):
+            table_data.append([
+                tensor_name,
+                stats.mean,
+                stats.max_abs,
+                stats.std,
+                stats.norm,
+                stats.numel
+            ])
+
+        if table_data:
+            table = wandb.Table(
+                columns=["tensor_name", "mean", "max_abs", "std", "norm", "numel"],
+                data=table_data
+            )
+            wandb.log({"activation_table": table})
+
+    def print_report(self, results: List[ModelResult]):
+        """Pretty-print comparison report."""
+        print("\n" + "=" * 110)
+        print(f"{'Model':<35} {'Loss':>10} {'PPL':>10} {'ActMean':>12} {'ActMaxAbs':>12} {'ActStd':>12} {'Tokens':>8}")
+        print("-" * 110)
+
+        for r in results:
+            name = r.model_name.split("/")[-1][:33]
+            print(
+                f"{name:<35} "
+                f"{r.loss:>10.4f} "
+                f"{r.perplexity:>10.2f} "
+                f"{r.global_act.mean:>12.6f} "
+                f"{r.global_act.max_abs:>12.6f} "
+                f"{r.global_act.std:>12.6f} "
+                f"{r.num_tokens:>8}"
+            )
+
+        print("=" * 110)
+
+        # Print top 5 layers by max_abs for each model
+        print("\n[Per-Tensor Max Abs Top 5]")
+        for r in results:
+            name = r.model_name.split("/")[-1]
+            sorted_layers = sorted(r.layer_acts.items(), key=lambda x: x[1].max_abs, reverse=True)[:5]
+            print(f"\n  {name}:")
+            for tensor_name, stats in sorted_layers:
+                print(f"    {tensor_name:<50} max_abs={stats.max_abs:>10.4f}  mean={stats.mean:>10.6f}  std={stats.std:>10.4f}")
+
+    def export_json(self, results: List[ModelResult], path: str):
+        """Export results to JSON."""
+        data = []
+        for r in results:
+            entry = {
+                "model": r.model_name,
+                "loss": r.loss,
+                "perplexity": r.perplexity,
+                "num_tokens": r.num_tokens,
+                "num_layers": r.num_layers,
+                "hidden_size": r.hidden_size,
+                "num_params": r.num_params,
+                "global_act": asdict(r.global_act),
+                "layer_acts": {k: asdict(v) for k, v in r.layer_acts.items()}
+            }
+            data.append(entry)
+
+        with open(path, "w") as f:
+            json.dump(data, f, indent=2)
+        print(f"[Exported] Results saved to {path}")

zain/Activation/out/glu-gelu-150L_run/training_log.jsonl

@@ -0,0 +1,2 @@
+{"step": 20, "epoch": 0.0013481631277384564, "timestamp": 1786558895.6596417, "loss": 7.965690612792969, "grad_norm": 1.2578125, "learning_rate": 0.000475, "train/total_time_seconds": 7.616961658000946, "train/time_per_step_avg": 0.3808480829000473, "train/epoch_time_elapsed": 8.680427201092243, "train/estimated_remaining_minutes": 6.22051868736744}
+{"step": 40, "epoch": 0.002696326255476913, "timestamp": 1786558903.827716, "loss": 6.832366943359375, "grad_norm": 0.87890625, "learning_rate": 0.0009750000000000001, "train/total_time_seconds": 14.811852153390646, "train/time_per_step_avg": 0.37029630383476614, "train/epoch_time_elapsed": 16.848502036184072, "train/estimated_remaining_minutes": 5.924740861356258}

zain/Activation/out/glu-relu-150L_run/checkpoint-1000/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "relu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-relu-150L_run/checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6035bf7964d463a92160083594168bbcb3e076204e33db15468d64c342a71de
+size 50427120

zain/Activation/out/glu-relu-150L_run/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f67562164598b7aeee7d46073b48a5f0c40fceda81d57eeed56d688e85d04f99
+size 101708778

zain/Activation/out/glu-relu-150L_run/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:645b71843375baf4cc10bc75a2f0f04e91b8e3f8e8929f518a87022898b3bc20
+size 14244

zain/Activation/out/glu-relu-150L_run/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9db76f238494a413266398860b344c6c4c0f3a045ef78b23af1afac6775a12d1
+size 1064

zain/Activation/out/glu-relu-150L_run/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-relu-150L_run/checkpoint-1000/trainer_state.json

@@ -0,0 +1,544 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.06740815638692282,
+  "eval_steps": 50,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.296875,
+      "learning_rate": 0.000475,
+      "loss": 7.964384460449219,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 0.8671875,
+      "learning_rate": 0.0009750000000000001,
+      "loss": 6.8277839660644535,
+      "step": 40
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 6.035980701446533,
+      "eval_runtime": 21.045,
+      "eval_samples_per_second": 452.697,
+      "eval_steps_per_second": 3.564,
+      "step": 50
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.20703125,
+      "learning_rate": 0.001475,
+      "loss": 6.076027679443359,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.193359375,
+      "learning_rate": 0.001975,
+      "loss": 6.007006454467773,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 0.79296875,
+      "learning_rate": 0.002475,
+      "loss": 5.9332275390625,
+      "step": 100
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 5.939233303070068,
+      "eval_runtime": 21.1007,
+      "eval_samples_per_second": 451.501,
+      "eval_steps_per_second": 3.554,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 0.2255859375,
+      "learning_rate": 0.0029749999999999998,
+      "loss": 5.783406066894531,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.703125,
+      "learning_rate": 0.003475,
+      "loss": 5.675531005859375,
+      "step": 140
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 5.526480674743652,
+      "eval_runtime": 21.379,
+      "eval_samples_per_second": 445.624,
+      "eval_steps_per_second": 3.508,
+      "step": 150
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.1943359375,
+      "learning_rate": 0.003975,
+      "loss": 5.528359603881836,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.2392578125,
+      "learning_rate": 0.004475,
+      "loss": 5.462315368652344,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.33203125,
+      "learning_rate": 0.004975,
+      "loss": 5.351004028320313,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 5.25657844543457,
+      "eval_runtime": 21.2496,
+      "eval_samples_per_second": 448.338,
+      "eval_steps_per_second": 3.529,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 1.3125,
+      "learning_rate": 0.005,
+      "loss": 5.367509841918945,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 0.255859375,
+      "learning_rate": 0.005,
+      "loss": 5.376996994018555,
+      "step": 240
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 5.037845134735107,
+      "eval_runtime": 21.089,
+      "eval_samples_per_second": 451.753,
+      "eval_steps_per_second": 3.556,
+      "step": 250
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 0.298828125,
+      "learning_rate": 0.005,
+      "loss": 5.0486492156982425,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.373046875,
+      "learning_rate": 0.005,
+      "loss": 4.887042999267578,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 0.404296875,
+      "learning_rate": 0.005,
+      "loss": 4.7291618347167965,
+      "step": 300
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "eval_loss": 4.651667594909668,
+      "eval_runtime": 21.1749,
+      "eval_samples_per_second": 449.919,
+      "eval_steps_per_second": 3.542,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 0.416015625,
+      "learning_rate": 0.005,
+      "loss": 4.579043960571289,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.37890625,
+      "learning_rate": 0.005,
+      "loss": 4.4609519958496096,
+      "step": 340
+    },
+    {
+      "epoch": 0.023592854735422986,
+      "eval_loss": 4.355501174926758,
+      "eval_runtime": 21.1941,
+      "eval_samples_per_second": 449.513,
+      "eval_steps_per_second": 3.539,
+      "step": 350
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 0.3984375,
+      "learning_rate": 0.005,
+      "loss": 4.370123672485351,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.390625,
+      "learning_rate": 0.005,
+      "loss": 4.307046890258789,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.447265625,
+      "learning_rate": 0.005,
+      "loss": 4.20123291015625,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 4.166988372802734,
+      "eval_runtime": 21.125,
+      "eval_samples_per_second": 450.982,
+      "eval_steps_per_second": 3.55,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.005,
+      "loss": 4.1340187072753904,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 0.6875,
+      "learning_rate": 0.005,
+      "loss": 4.119440078735352,
+      "step": 440
+    },
+    {
+      "epoch": 0.030333670374115267,
+      "eval_loss": 4.067277431488037,
+      "eval_runtime": 21.0533,
+      "eval_samples_per_second": 452.519,
+      "eval_steps_per_second": 3.562,
+      "step": 450
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.62109375,
+      "learning_rate": 0.005,
+      "loss": 4.072461318969727,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 1.4375,
+      "learning_rate": 0.005,
+      "loss": 4.022983551025391,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.7890625,
+      "learning_rate": 0.005,
+      "loss": 4.041971969604492,
+      "step": 500
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "eval_loss": 4.022434234619141,
+      "eval_runtime": 21.1962,
+      "eval_samples_per_second": 449.468,
+      "eval_steps_per_second": 3.538,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 3.0,
+      "learning_rate": 0.005,
+      "loss": 4.0136863708496096,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.46484375,
+      "learning_rate": 0.005,
+      "loss": 3.9774227142333984,
+      "step": 540
+    },
+    {
+      "epoch": 0.03707448601280755,
+      "eval_loss": 3.9099950790405273,
+      "eval_runtime": 21.1739,
+      "eval_samples_per_second": 449.94,
+      "eval_steps_per_second": 3.542,
+      "step": 550
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.435546875,
+      "learning_rate": 0.005,
+      "loss": 3.916512298583984,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.5546875,
+      "learning_rate": 0.005,
+      "loss": 3.9239063262939453,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.494140625,
+      "learning_rate": 0.005,
+      "loss": 3.886766052246094,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 3.8580801486968994,
+      "eval_runtime": 21.3144,
+      "eval_samples_per_second": 446.975,
+      "eval_steps_per_second": 3.519,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.78515625,
+      "learning_rate": 0.005,
+      "loss": 3.851901626586914,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.51953125,
+      "learning_rate": 0.005,
+      "loss": 3.8467464447021484,
+      "step": 640
+    },
+    {
+      "epoch": 0.043815301651499834,
+      "eval_loss": 3.7791011333465576,
+      "eval_runtime": 21.1038,
+      "eval_samples_per_second": 451.436,
+      "eval_steps_per_second": 3.554,
+      "step": 650
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 0.5234375,
+      "learning_rate": 0.005,
+      "loss": 3.7780517578125,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 0.462890625,
+      "learning_rate": 0.005,
+      "loss": 3.8014129638671874,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 1.4375,
+      "learning_rate": 0.005,
+      "loss": 3.7913097381591796,
+      "step": 700
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "eval_loss": 3.8076748847961426,
+      "eval_runtime": 21.2634,
+      "eval_samples_per_second": 448.047,
+      "eval_steps_per_second": 3.527,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.455078125,
+      "learning_rate": 0.005,
+      "loss": 3.7911834716796875,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.70703125,
+      "learning_rate": 0.005,
+      "loss": 3.786530303955078,
+      "step": 740
+    },
+    {
+      "epoch": 0.05055611729019211,
+      "eval_loss": 3.783113479614258,
+      "eval_runtime": 21.2307,
+      "eval_samples_per_second": 448.738,
+      "eval_steps_per_second": 3.533,
+      "step": 750
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.59765625,
+      "learning_rate": 0.005,
+      "loss": 3.766161346435547,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 1.4921875,
+      "learning_rate": 0.005,
+      "loss": 3.8679542541503906,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 1.140625,
+      "learning_rate": 0.005,
+      "loss": 3.820303726196289,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 3.8336594104766846,
+      "eval_runtime": 21.0369,
+      "eval_samples_per_second": 452.871,
+      "eval_steps_per_second": 3.565,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 3.078125,
+      "learning_rate": 0.005,
+      "loss": 3.9229141235351563,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.8359375,
+      "learning_rate": 0.005,
+      "loss": 3.974406051635742,
+      "step": 840
+    },
+    {
+      "epoch": 0.057296932928884395,
+      "eval_loss": 4.010101318359375,
+      "eval_runtime": 21.2469,
+      "eval_samples_per_second": 448.394,
+      "eval_steps_per_second": 3.53,
+      "step": 850
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 1.3984375,
+      "learning_rate": 0.005,
+      "loss": 4.004122543334961,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 1.46875,
+      "learning_rate": 0.005,
+      "loss": 4.010852432250976,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 1.671875,
+      "learning_rate": 0.005,
+      "loss": 4.027272796630859,
+      "step": 900
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "eval_loss": 4.2301554679870605,
+      "eval_runtime": 21.1781,
+      "eval_samples_per_second": 449.851,
+      "eval_steps_per_second": 3.541,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 2.25,
+      "learning_rate": 0.005,
+      "loss": 4.1036529541015625,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 1.6328125,
+      "learning_rate": 0.005,
+      "loss": 4.183735656738281,
+      "step": 940
+    },
+    {
+      "epoch": 0.06403774856757667,
+      "eval_loss": 4.2052483558654785,
+      "eval_runtime": 21.0112,
+      "eval_samples_per_second": 453.425,
+      "eval_steps_per_second": 3.57,
+      "step": 950
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 1.5078125,
+      "learning_rate": 0.005,
+      "loss": 4.163127899169922,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 1.84375,
+      "learning_rate": 0.005,
+      "loss": 4.166858291625976,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 1.4296875,
+      "learning_rate": 0.005,
+      "loss": 4.1427558898925785,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 4.123730659484863,
+      "eval_runtime": 21.1825,
+      "eval_samples_per_second": 449.759,
+      "eval_steps_per_second": 3.541,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4839413121024000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-relu-150L_run/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ce5601cb1d765f72452322314a16a412136c90a598f703356047035624bc3e5
+size 4920

zain/Activation/out/glu-relu-150L_run/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "relu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-relu-150L_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6035bf7964d463a92160083594168bbcb3e076204e33db15468d64c342a71de
+size 50427120

zain/Activation/out/glu-relu-150L_run/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-relu-150L_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ce5601cb1d765f72452322314a16a412136c90a598f703356047035624bc3e5
+size 4920

zain/Activation/out/glu-relu-150L_run/training_log.jsonl

@@ -0,0 +1,72 @@
+{"step": 20, "epoch": 0.0013481631277384564, "timestamp": 1786558042.565657, "loss": 7.964384460449219, "grad_norm": 1.296875, "learning_rate": 0.000475, "train/total_time_seconds": 7.479156076908112, "train/time_per_step_avg": 0.37395780384540556, "train/epoch_time_elapsed": 8.524832468479872, "train/estimated_remaining_minutes": 6.107977462808291}
+{"step": 40, "epoch": 0.002696326255476913, "timestamp": 1786558050.5149207, "loss": 6.8277839660644535, "grad_norm": 0.8671875, "learning_rate": 0.0009750000000000001, "train/total_time_seconds": 14.462965689599514, "train/time_per_step_avg": 0.36157414223998785, "train/epoch_time_elapsed": 16.474097702652216, "train/estimated_remaining_minutes": 5.785186275839806}
+{"step": 50, "epoch": 0.003370407819346141, "timestamp": 1786558075.5407507, "eval_loss": 6.035980701446533, "eval_runtime": 21.045, "eval_samples_per_second": 452.697, "eval_steps_per_second": 3.564, "train/total_time_seconds": 17.960122998803854, "train/time_per_step_avg": 0.3592024599760771, "train/epoch_time_elapsed": 41.49992706999183, "train/estimated_remaining_minutes": 5.687372282954554}
+{"step": 60, "epoch": 0.004044489383215369, "timestamp": 1786558079.519231, "loss": 6.076027679443359, "grad_norm": 0.20703125, "learning_rate": 0.001475, "train/total_time_seconds": 21.45620109140873, "train/time_per_step_avg": 0.35760335152347883, "train/epoch_time_elapsed": 45.478407979011536, "train/estimated_remaining_minutes": 5.602452507201168}
+{"step": 80, "epoch": 0.005392652510953826, "timestamp": 1786558087.4979699, "loss": 6.007006454467773, "grad_norm": 0.193359375, "learning_rate": 0.001975, "train/total_time_seconds": 28.471343249082565, "train/time_per_step_avg": 0.35589179061353204, "train/epoch_time_elapsed": 53.457146506756544, "train/estimated_remaining_minutes": 5.457007456074158}
+{"step": 100, "epoch": 0.006740815638692282, "timestamp": 1786558095.48223, "loss": 5.9332275390625, "grad_norm": 0.79296875, "learning_rate": 0.002475, "train/total_time_seconds": 35.47478394955397, "train/time_per_step_avg": 0.3547478394955397, "train/epoch_time_elapsed": 61.44140635803342, "train/estimated_remaining_minutes": 5.321217592433095}
+{"step": 100, "epoch": 0.006740815638692282, "timestamp": 1786558116.5845, "eval_loss": 5.939233303070068, "eval_runtime": 21.1007, "eval_samples_per_second": 451.501, "eval_steps_per_second": 3.554, "train/total_time_seconds": 35.47478394955397, "train/time_per_step_avg": 0.3547478394955397, "train/epoch_time_elapsed": 82.54367513209581, "train/estimated_remaining_minutes": 5.321217592433095}
+{"step": 120, "epoch": 0.008088978766430738, "timestamp": 1786558124.596924, "loss": 5.783406066894531, "grad_norm": 0.2255859375, "learning_rate": 0.0029749999999999998, "train/total_time_seconds": 42.49194718524814, "train/time_per_step_avg": 0.35012791108340025, "train/epoch_time_elapsed": 90.55610039830208, "train/estimated_remaining_minutes": 5.193460211530327}
+{"step": 140, "epoch": 0.009437141894169195, "timestamp": 1786558132.8711853, "loss": 5.675531005859375, "grad_norm": 0.703125, "learning_rate": 0.003475, "train/total_time_seconds": 49.799739610403776, "train/time_per_step_avg": 0.3533677392080426, "train/epoch_time_elapsed": 98.83036207780242, "train/estimated_remaining_minutes": 5.098544769636577}
+{"step": 150, "epoch": 0.010111223458038422, "timestamp": 1786558158.2318413, "eval_loss": 5.526480674743652, "eval_runtime": 21.379, "eval_samples_per_second": 445.624, "eval_steps_per_second": 3.508, "train/total_time_seconds": 53.29423313960433, "train/time_per_step_avg": 0.3533411014080048, "train/epoch_time_elapsed": 124.1910157725215, "train/estimated_remaining_minutes": 5.033344240962632}
+{"step": 160, "epoch": 0.010785305021907651, "timestamp": 1786558162.2187576, "loss": 5.528359603881836, "grad_norm": 0.1943359375, "learning_rate": 0.003975, "train/total_time_seconds": 56.78381925448775, "train/time_per_step_avg": 0.3532761816307902, "train/epoch_time_elapsed": 128.17793262004852, "train/estimated_remaining_minutes": 4.968584184767678}
+{"step": 180, "epoch": 0.012133468149646108, "timestamp": 1786558170.192914, "loss": 5.462315368652344, "grad_norm": 0.2392578125, "learning_rate": 0.004475, "train/total_time_seconds": 63.79622644558549, "train/time_per_step_avg": 0.35324883196502926, "train/epoch_time_elapsed": 136.1520889699459, "train/estimated_remaining_minutes": 4.84378756346112}
+{"step": 200, "epoch": 0.013481631277384564, "timestamp": 1786558178.19487, "loss": 5.351004028320313, "grad_norm": 0.33203125, "learning_rate": 0.004975, "train/total_time_seconds": 70.81487478688359, "train/time_per_step_avg": 0.35340090837329624, "train/epoch_time_elapsed": 144.15404615923762, "train/estimated_remaining_minutes": 4.720991652458906}
+{"step": 200, "epoch": 0.013481631277384564, "timestamp": 1786558199.4460309, "eval_loss": 5.25657844543457, "eval_runtime": 21.2496, "eval_samples_per_second": 448.338, "eval_steps_per_second": 3.529, "train/total_time_seconds": 70.81487478688359, "train/time_per_step_avg": 0.35340090837329624, "train/epoch_time_elapsed": 165.40520602837205, "train/estimated_remaining_minutes": 4.720991652458906}
+{"step": 220, "epoch": 0.01482979440512302, "timestamp": 1786558207.49547, "loss": 5.367509841918945, "grad_norm": 1.3125, "learning_rate": 0.005, "train/total_time_seconds": 77.85458770766854, "train/time_per_step_avg": 0.3536264052242041, "train/epoch_time_elapsed": 173.45464695617557, "train/estimated_remaining_minutes": 4.60049836454405}
+{"step": 240, "epoch": 0.016177957532861477, "timestamp": 1786558215.5625825, "loss": 5.376996994018555, "grad_norm": 0.255859375, "learning_rate": 0.005, "train/total_time_seconds": 84.92369622737169, "train/time_per_step_avg": 0.35123956616967916, "train/epoch_time_elapsed": 181.5217593461275, "train/estimated_remaining_minutes": 4.482083967555728}
+{"step": 250, "epoch": 0.016852039096730706, "timestamp": 1786558240.6273065, "eval_loss": 5.037845134735107, "eval_runtime": 21.089, "eval_samples_per_second": 451.753, "eval_steps_per_second": 3.556, "train/total_time_seconds": 88.41916769742966, "train/time_per_step_avg": 0.35124934557825327, "train/epoch_time_elapsed": 206.5864817649126, "train/estimated_remaining_minutes": 4.420958384871483}
+{"step": 260, "epoch": 0.01752612066059993, "timestamp": 1786558244.673984, "loss": 5.0486492156982425, "grad_norm": 0.298828125, "learning_rate": 0.005, "train/total_time_seconds": 91.93673975765705, "train/time_per_step_avg": 0.351529205031693, "train/epoch_time_elapsed": 210.63316006585956, "train/estimated_remaining_minutes": 4.361101757735014}
+{"step": 280, "epoch": 0.01887428378833839, "timestamp": 1786558252.9377697, "loss": 4.887042999267578, "grad_norm": 0.373046875, "learning_rate": 0.005, "train/total_time_seconds": 99.2333196401596, "train/time_per_step_avg": 0.35437093194574115, "train/epoch_time_elapsed": 218.89694626629353, "train/estimated_remaining_minutes": 4.25285655600684}
+{"step": 300, "epoch": 0.020222446916076844, "timestamp": 1786558260.8977244, "loss": 4.7291618347167965, "grad_norm": 0.404296875, "learning_rate": 0.005, "train/total_time_seconds": 106.21757336333394, "train/time_per_step_avg": 0.3540269857645035, "train/epoch_time_elapsed": 226.85690063610673, "train/estimated_remaining_minutes": 4.1306834085740975}
+{"step": 300, "epoch": 0.020222446916076844, "timestamp": 1786558282.0742264, "eval_loss": 4.651667594909668, "eval_runtime": 21.1749, "eval_samples_per_second": 449.919, "eval_steps_per_second": 3.542, "train/total_time_seconds": 106.21757336333394, "train/time_per_step_avg": 0.3540269857645035, "train/epoch_time_elapsed": 248.03340182080865, "train/estimated_remaining_minutes": 4.1306834085740975}
+{"step": 320, "epoch": 0.021570610043815303, "timestamp": 1786558290.040653, "loss": 4.579043960571289, "grad_norm": 0.416015625, "learning_rate": 0.005, "train/total_time_seconds": 113.2087592817843, "train/time_per_step_avg": 0.35354171574115756, "train/epoch_time_elapsed": 255.99982998892665, "train/estimated_remaining_minutes": 4.009476891229861}
+{"step": 340, "epoch": 0.022918773171553757, "timestamp": 1786558298.1039598, "loss": 4.4609519958496096, "grad_norm": 0.37890625, "learning_rate": 0.005, "train/total_time_seconds": 120.27493808791041, "train/time_per_step_avg": 0.35351241860538724, "train/epoch_time_elapsed": 264.0631362423301, "train/estimated_remaining_minutes": 3.8912479969618077}
+{"step": 350, "epoch": 0.023592854735422986, "timestamp": 1786558323.2715254, "eval_loss": 4.355501174926758, "eval_runtime": 21.1941, "eval_samples_per_second": 449.513, "eval_steps_per_second": 3.539, "train/total_time_seconds": 123.76323676109314, "train/time_per_step_avg": 0.3534406906366348, "train/epoch_time_elapsed": 289.2306997999549, "train/estimated_remaining_minutes": 3.830766852129073}
+{"step": 360, "epoch": 0.024266936299292215, "timestamp": 1786558327.2571225, "loss": 4.370123672485351, "grad_norm": 0.3984375, "learning_rate": 0.005, "train/total_time_seconds": 127.26498315483332, "train/time_per_step_avg": 0.3532824339717627, "train/epoch_time_elapsed": 293.2162990421057, "train/estimated_remaining_minutes": 3.770814315698765}
+{"step": 380, "epoch": 0.02561509942703067, "timestamp": 1786558335.3185263, "loss": 4.307046890258789, "grad_norm": 0.390625, "learning_rate": 0.005, "train/total_time_seconds": 134.31174723431468, "train/time_per_step_avg": 0.3507842759415507, "train/epoch_time_elapsed": 301.27770306915045, "train/estimated_remaining_minutes": 3.6523369861962762}
+{"step": 400, "epoch": 0.026963262554769128, "timestamp": 1786558343.6275973, "loss": 4.20123291015625, "grad_norm": 0.447265625, "learning_rate": 0.005, "train/total_time_seconds": 141.64740459993482, "train/time_per_step_avg": 0.3542983123660088, "train/epoch_time_elapsed": 309.58677385374904, "train/estimated_remaining_minutes": 3.5411851149983704}
+{"step": 400, "epoch": 0.026963262554769128, "timestamp": 1786558364.7541769, "eval_loss": 4.166988372802734, "eval_runtime": 21.125, "eval_samples_per_second": 450.982, "eval_steps_per_second": 3.55, "train/total_time_seconds": 141.64740459993482, "train/time_per_step_avg": 0.3542983123660088, "train/epoch_time_elapsed": 330.71335170790553, "train/estimated_remaining_minutes": 3.5411851149983704}
+{"step": 420, "epoch": 0.028311425682507583, "timestamp": 1786558372.7547848, "loss": 4.1340187072753904, "grad_norm": 0.515625, "learning_rate": 0.005, "train/total_time_seconds": 148.66470146924257, "train/time_per_step_avg": 0.3545594218745828, "train/epoch_time_elapsed": 338.7139613032341, "train/estimated_remaining_minutes": 3.421647890958758}
+{"step": 440, "epoch": 0.02965958881024604, "timestamp": 1786558380.761076, "loss": 4.119440078735352, "grad_norm": 0.6875, "learning_rate": 0.005, "train/total_time_seconds": 155.69208113476634, "train/time_per_step_avg": 0.35417143046855926, "train/epoch_time_elapsed": 346.72025215625763, "train/estimated_remaining_minutes": 3.3025592967980737}
+{"step": 450, "epoch": 0.030333670374115267, "timestamp": 1786558405.7770348, "eval_loss": 4.067277431488037, "eval_runtime": 21.0533, "eval_samples_per_second": 452.519, "eval_steps_per_second": 3.562, "train/total_time_seconds": 159.17676619067788, "train/time_per_step_avg": 0.3541352942958474, "train/epoch_time_elapsed": 371.73621083050966, "train/estimated_remaining_minutes": 3.242489681661957}
+{"step": 460, "epoch": 0.031007751937984496, "timestamp": 1786558409.7575188, "loss": 4.072461318969727, "grad_norm": 0.62109375, "learning_rate": 0.005, "train/total_time_seconds": 162.67413600161672, "train/time_per_step_avg": 0.354091528467834, "train/epoch_time_elapsed": 375.7166953384876, "train/estimated_remaining_minutes": 3.18275483481424}
+{"step": 480, "epoch": 0.032355915065722954, "timestamp": 1786558417.6906242, "loss": 4.022983551025391, "grad_norm": 1.4375, "learning_rate": 0.005, "train/total_time_seconds": 169.6476893760264, "train/time_per_step_avg": 0.3533594214171171, "train/epoch_time_elapsed": 383.6498010158539, "train/estimated_remaining_minutes": 3.0630832804004764}
+{"step": 500, "epoch": 0.03370407819346141, "timestamp": 1786558425.6347263, "loss": 4.041971969604492, "grad_norm": 0.7890625, "learning_rate": 0.005, "train/total_time_seconds": 176.62393491342664, "train/time_per_step_avg": 0.3497653031349182, "train/epoch_time_elapsed": 391.59390341863036, "train/estimated_remaining_minutes": 2.9437322485571107}
+{"step": 500, "epoch": 0.03370407819346141, "timestamp": 1786558446.8324366, "eval_loss": 4.022434234619141, "eval_runtime": 21.1962, "eval_samples_per_second": 449.468, "eval_steps_per_second": 3.538, "train/total_time_seconds": 176.62393491342664, "train/time_per_step_avg": 0.3497653031349182, "train/epoch_time_elapsed": 412.7916120700538, "train/estimated_remaining_minutes": 2.9437322485571107}
+{"step": 520, "epoch": 0.03505224132119986, "timestamp": 1786558455.0041444, "loss": 4.0136863708496096, "grad_norm": 3.0, "learning_rate": 0.005, "train/total_time_seconds": 183.8067372404039, "train/time_per_step_avg": 0.3514203577116132, "train/epoch_time_elapsed": 420.963321775198, "train/estimated_remaining_minutes": 2.827795957544675}
+{"step": 540, "epoch": 0.03640040444893832, "timestamp": 1786558463.4831052, "loss": 3.9774227142333984, "grad_norm": 0.46484375, "learning_rate": 0.005, "train/total_time_seconds": 191.26865224540234, "train/time_per_step_avg": 0.35576571110635996, "train/epoch_time_elapsed": 429.4422825425863, "train/estimated_remaining_minutes": 2.715542593607564}
+{"step": 550, "epoch": 0.03707448601280755, "timestamp": 1786558488.7636483, "eval_loss": 3.9099950790405273, "eval_runtime": 21.1739, "eval_samples_per_second": 449.94, "eval_steps_per_second": 3.542, "train/total_time_seconds": 194.88533645495772, "train/time_per_step_avg": 0.35708570264279843, "train/epoch_time_elapsed": 454.72282434627414, "train/estimated_remaining_minutes": 2.657527315294878}
+{"step": 560, "epoch": 0.03774856757667678, "timestamp": 1786558492.8987718, "loss": 3.916512298583984, "grad_norm": 0.435546875, "learning_rate": 0.005, "train/total_time_seconds": 198.52793854102492, "train/time_per_step_avg": 0.3585380253940821, "train/epoch_time_elapsed": 458.85794903337955, "train/estimated_remaining_minutes": 2.599770623751517}
+{"step": 580, "epoch": 0.03909673070441524, "timestamp": 1786558501.1090071, "loss": 3.9239063262939453, "grad_norm": 0.5546875, "learning_rate": 0.005, "train/total_time_seconds": 205.76045013964176, "train/time_per_step_avg": 0.3611276076361537, "train/epoch_time_elapsed": 467.0681842379272, "train/estimated_remaining_minutes": 2.483315777547401}
+{"step": 600, "epoch": 0.04044489383215369, "timestamp": 1786558509.3264072, "loss": 3.886766052246094, "grad_norm": 0.494140625, "learning_rate": 0.005, "train/total_time_seconds": 212.9962049201131, "train/time_per_step_avg": 0.36372270006686447, "train/epoch_time_elapsed": 475.2855847887695, "train/estimated_remaining_minutes": 2.3666244991123677}
+{"step": 600, "epoch": 0.04044489383215369, "timestamp": 1786558530.642233, "eval_loss": 3.8580801486968994, "eval_runtime": 21.3144, "eval_samples_per_second": 446.975, "eval_steps_per_second": 3.519, "train/total_time_seconds": 212.9962049201131, "train/time_per_step_avg": 0.36372270006686447, "train/epoch_time_elapsed": 496.60140960663557, "train/estimated_remaining_minutes": 2.3666244991123677}
+{"step": 620, "epoch": 0.04179305695989215, "timestamp": 1786558538.7886603, "loss": 3.851901626586914, "grad_norm": 0.78515625, "learning_rate": 0.005, "train/total_time_seconds": 220.16741828620434, "train/time_per_step_avg": 0.36360681045800447, "train/epoch_time_elapsed": 504.7478371486068, "train/estimated_remaining_minutes": 2.24902201475155}
+{"step": 640, "epoch": 0.043141220087630605, "timestamp": 1786558546.9965365, "loss": 3.8467464447021484, "grad_norm": 0.51953125, "learning_rate": 0.005, "train/total_time_seconds": 227.3981082253158, "train/time_per_step_avg": 0.3612945597991347, "train/epoch_time_elapsed": 512.9557139910758, "train/estimated_remaining_minutes": 2.1318572646123357}
+{"step": 650, "epoch": 0.043815301651499834, "timestamp": 1786558572.2321274, "eval_loss": 3.7791011333465576, "eval_runtime": 21.1038, "eval_samples_per_second": 451.436, "eval_steps_per_second": 3.554, "train/total_time_seconds": 231.0267353616655, "train/time_per_step_avg": 0.3614139890670776, "train/epoch_time_elapsed": 538.1913029253483, "train/estimated_remaining_minutes": 2.0733168558098187}
+{"step": 660, "epoch": 0.044489383215369056, "timestamp": 1786558576.3355627, "loss": 3.7780517578125, "grad_norm": 0.5234375, "learning_rate": 0.005, "train/total_time_seconds": 234.64380882307887, "train/time_per_step_avg": 0.36115870282053947, "train/epoch_time_elapsed": 542.2947395294905, "train/estimated_remaining_minutes": 2.0146185606021922}
+{"step": 680, "epoch": 0.045837546343107514, "timestamp": 1786558584.7434273, "loss": 3.8014129638671874, "grad_norm": 0.462890625, "learning_rate": 0.005, "train/total_time_seconds": 242.05922985076904, "train/time_per_step_avg": 0.3629877971112728, "train/epoch_time_elapsed": 550.702604662627, "train/estimated_remaining_minutes": 1.8985037635354436}
+{"step": 700, "epoch": 0.04718570947084597, "timestamp": 1786558592.9433744, "loss": 3.7913097381591796, "grad_norm": 1.4375, "learning_rate": 0.005, "train/total_time_seconds": 249.2656230814755, "train/time_per_step_avg": 0.3626941816136241, "train/epoch_time_elapsed": 558.9025516286492, "train/estimated_remaining_minutes": 1.7804687362962535}
+{"step": 700, "epoch": 0.04718570947084597, "timestamp": 1786558614.2083035, "eval_loss": 3.8076748847961426, "eval_runtime": 21.2634, "eval_samples_per_second": 448.047, "eval_steps_per_second": 3.527, "train/total_time_seconds": 249.2656230814755, "train/time_per_step_avg": 0.3626941816136241, "train/epoch_time_elapsed": 580.1674793735147, "train/estimated_remaining_minutes": 1.7804687362962535}
+{"step": 720, "epoch": 0.04853387259858443, "timestamp": 1786558622.484525, "loss": 3.7911834716796875, "grad_norm": 0.455078125, "learning_rate": 0.005, "train/total_time_seconds": 256.5397740714252, "train/time_per_step_avg": 0.3637235578522086, "train/epoch_time_elapsed": 588.4437025524676, "train/estimated_remaining_minutes": 1.6627577949073855}
+{"step": 740, "epoch": 0.04988203572632288, "timestamp": 1786558630.6349056, "loss": 3.786530303955078, "grad_norm": 0.70703125, "learning_rate": 0.005, "train/total_time_seconds": 263.719080761075, "train/time_per_step_avg": 0.3632097253575921, "train/epoch_time_elapsed": 596.5940828919411, "train/estimated_remaining_minutes": 1.5443009233756646}
+{"step": 750, "epoch": 0.05055611729019211, "timestamp": 1786558655.9304035, "eval_loss": 3.783113479614258, "eval_runtime": 21.2307, "eval_samples_per_second": 448.738, "eval_steps_per_second": 3.533, "train/total_time_seconds": 267.30078534036875, "train/time_per_step_avg": 0.3627404997870326, "train/epoch_time_elapsed": 621.8895792737603, "train/estimated_remaining_minutes": 1.4850043630020484}
+{"step": 760, "epoch": 0.05123019885406134, "timestamp": 1786558659.9095356, "loss": 3.766161346435547, "grad_norm": 0.59765625, "learning_rate": 0.005, "train/total_time_seconds": 270.7942376807332, "train/time_per_step_avg": 0.3615042885765433, "train/epoch_time_elapsed": 625.8687125109136, "train/estimated_remaining_minutes": 1.425232829898596}
+{"step": 780, "epoch": 0.0525783619817998, "timestamp": 1786558667.869698, "loss": 3.8679542541503906, "grad_norm": 1.4921875, "learning_rate": 0.005, "train/total_time_seconds": 277.78589740768075, "train/time_per_step_avg": 0.35726667556911706, "train/epoch_time_elapsed": 633.8288739770651, "train/estimated_remaining_minutes": 1.3058311416600377}
+{"step": 800, "epoch": 0.053926525109538256, "timestamp": 1786558675.8750415, "loss": 3.820303726196289, "grad_norm": 1.140625, "learning_rate": 0.005, "train/total_time_seconds": 284.8019813261926, "train/time_per_step_avg": 0.3553635824471712, "train/epoch_time_elapsed": 641.8342174440622, "train/estimated_remaining_minutes": 1.1866749221924693}
+{"step": 800, "epoch": 0.053926525109538256, "timestamp": 1786558696.9135675, "eval_loss": 3.8336594104766846, "eval_runtime": 21.0369, "eval_samples_per_second": 452.871, "eval_steps_per_second": 3.565, "train/total_time_seconds": 284.8019813261926, "train/time_per_step_avg": 0.3553635824471712, "train/epoch_time_elapsed": 662.8727425672114, "train/estimated_remaining_minutes": 1.1866749221924693}
+{"step": 820, "epoch": 0.05527468823727671, "timestamp": 1786558705.2758486, "loss": 3.9229141235351563, "grad_norm": 3.078125, "learning_rate": 0.005, "train/total_time_seconds": 292.17277770489454, "train/time_per_step_avg": 0.35633003633469346, "train/epoch_time_elapsed": 671.2350256331265, "train/estimated_remaining_minutes": 1.0689247964813215}
+{"step": 840, "epoch": 0.056622851365015166, "timestamp": 1786558713.3453393, "loss": 3.974406051635742, "grad_norm": 0.8359375, "learning_rate": 0.005, "train/total_time_seconds": 299.2718831561506, "train/time_per_step_avg": 0.3555280239507556, "train/epoch_time_elapsed": 679.3045146428049, "train/estimated_remaining_minutes": 0.950069470336986}
+{"step": 850, "epoch": 0.057296932928884395, "timestamp": 1786558738.5791726, "eval_loss": 4.010101318359375, "eval_runtime": 21.2469, "eval_samples_per_second": 448.394, "eval_steps_per_second": 3.53, "train/total_time_seconds": 302.77213901653886, "train/time_per_step_avg": 0.3547135367617011, "train/epoch_time_elapsed": 704.5383482053876, "train/estimated_remaining_minutes": 0.8905062912251142}
+{"step": 860, "epoch": 0.057971014492753624, "timestamp": 1786558742.6643817, "loss": 4.004122543334961, "grad_norm": 1.3984375, "learning_rate": 0.005, "train/total_time_seconds": 306.37070309743285, "train/time_per_step_avg": 0.3557646541669965, "train/epoch_time_elapsed": 708.6235591545701, "train/estimated_remaining_minutes": 0.8312383417372209}
+{"step": 880, "epoch": 0.05931917762049208, "timestamp": 1786558750.841112, "loss": 4.010852432250976, "grad_norm": 1.46875, "learning_rate": 0.005, "train/total_time_seconds": 313.57940961420536, "train/time_per_step_avg": 0.3579351220652461, "train/epoch_time_elapsed": 716.8002890385687, "train/estimated_remaining_minutes": 0.7126804763959212}
+{"step": 900, "epoch": 0.06066734074823053, "timestamp": 1786558759.0011556, "loss": 4.027272796630859, "grad_norm": 1.671875, "learning_rate": 0.005, "train/total_time_seconds": 320.76208513975143, "train/time_per_step_avg": 0.3596010381355882, "train/epoch_time_elapsed": 724.9603324122727, "train/estimated_remaining_minutes": 0.59400386136991}
+{"step": 900, "epoch": 0.06066734074823053, "timestamp": 1786558780.1807094, "eval_loss": 4.2301554679870605, "eval_runtime": 21.1781, "eval_samples_per_second": 449.851, "eval_steps_per_second": 3.541, "train/total_time_seconds": 320.76208513975143, "train/time_per_step_avg": 0.3596010381355882, "train/epoch_time_elapsed": 746.1398853138089, "train/estimated_remaining_minutes": 0.59400386136991}
+{"step": 920, "epoch": 0.06201550387596899, "timestamp": 1786558788.2090843, "loss": 4.1036529541015625, "grad_norm": 2.25, "learning_rate": 0.005, "train/total_time_seconds": 327.8070343770087, "train/time_per_step_avg": 0.35634256672114134, "train/epoch_time_elapsed": 754.1682606600225, "train/estimated_remaining_minutes": 0.4750826585174039}
+{"step": 940, "epoch": 0.06336366700370745, "timestamp": 1786558796.2388895, "loss": 4.183735656738281, "grad_norm": 1.6328125, "learning_rate": 0.005, "train/total_time_seconds": 334.84725157916546, "train/time_per_step_avg": 0.3557536842301488, "train/epoch_time_elapsed": 762.1980660632253, "train/estimated_remaining_minutes": 0.3562204804033675}
+{"step": 950, "epoch": 0.06403774856757667, "timestamp": 1786558821.6363292, "eval_loss": 4.2052483558654785, "eval_runtime": 21.0112, "eval_samples_per_second": 453.425, "eval_steps_per_second": 3.57, "train/total_time_seconds": 338.7223717160523, "train/time_per_step_avg": 0.35950232699513435, "train/epoch_time_elapsed": 787.5955048166215, "train/estimated_remaining_minutes": 0.29712488747022126}
+{"step": 960, "epoch": 0.06471183013144591, "timestamp": 1786558825.6111045, "loss": 4.163127899169922, "grad_norm": 1.5078125, "learning_rate": 0.005, "train/total_time_seconds": 342.21410282701254, "train/time_per_step_avg": 0.3584339972957969, "train/epoch_time_elapsed": 791.570281419903, "train/estimated_remaining_minutes": 0.23764868251875867}
+{"step": 980, "epoch": 0.06605999325918437, "timestamp": 1786558833.5979323, "loss": 4.166858291625976, "grad_norm": 1.84375, "learning_rate": 0.005, "train/total_time_seconds": 349.22664188221097, "train/time_per_step_avg": 0.3564723226800561, "train/epoch_time_elapsed": 799.5571090690792, "train/estimated_remaining_minutes": 0.11878457206877924}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786558841.5874877, "loss": 4.1427558898925785, "grad_norm": 1.4296875, "learning_rate": 0.005, "train/total_time_seconds": 356.23444191366434, "train/time_per_step_avg": 0.35472356773912905, "train/epoch_time_elapsed": 807.5466644875705, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786558862.7715588, "eval_loss": 4.123730659484863, "eval_runtime": 21.1825, "eval_samples_per_second": 449.759, "eval_steps_per_second": 3.541, "train/total_time_seconds": 356.23444191366434, "train/time_per_step_avg": 0.35472356773912905, "train/epoch_time_elapsed": 828.7307346388698, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786558863.2290063, "train_runtime": 829.8809, "train_samples_per_second": 77.12, "train_steps_per_second": 1.205, "total_flos": 4839413121024000.0, "train_loss": 4.536944602966309, "train/total_time_seconds": 356.23444191366434, "train/time_per_step_avg": 0.35472356773912905, "train/epoch_time_elapsed": 829.1881820075214, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786558884.299808, "eval_loss": 4.123730659484863, "eval_runtime": 21.0674, "eval_samples_per_second": 452.215, "eval_steps_per_second": 3.56, "train/total_time_seconds": 356.23444191366434, "train/time_per_step_avg": 0.35472356773912905, "train/epoch_time_elapsed": 850.2589839994907, "train/estimated_remaining_minutes": 0.0}

zain/Activation/out/glu-silu-150L_run/checkpoint-1000/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-150L_run/checkpoint-1000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c19d4f22bca1d6c50b4226f59e84e12d53357840e7cb70cf90784aefb45a5314
+size 50427120

zain/Activation/out/glu-silu-150L_run/checkpoint-1000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57d2993745383b14d2c6ad536a710652d576adb850224cf1ab05fa0a97b04c9c
+size 101708778

zain/Activation/out/glu-silu-150L_run/checkpoint-1000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:645b71843375baf4cc10bc75a2f0f04e91b8e3f8e8929f518a87022898b3bc20
+size 14244

zain/Activation/out/glu-silu-150L_run/checkpoint-1000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9db76f238494a413266398860b344c6c4c0f3a045ef78b23af1afac6775a12d1
+size 1064

zain/Activation/out/glu-silu-150L_run/checkpoint-1000/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-150L_run/checkpoint-1000/trainer_state.json

@@ -0,0 +1,544 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.06740815638692282,
+  "eval_steps": 50,
+  "global_step": 1000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0013481631277384564,
+      "grad_norm": 1.2578125,
+      "learning_rate": 0.000475,
+      "loss": 7.965912628173828,
+      "step": 20
+    },
+    {
+      "epoch": 0.002696326255476913,
+      "grad_norm": 0.875,
+      "learning_rate": 0.0009750000000000001,
+      "loss": 6.833769226074219,
+      "step": 40
+    },
+    {
+      "epoch": 0.003370407819346141,
+      "eval_loss": 6.036951541900635,
+      "eval_runtime": 20.9461,
+      "eval_samples_per_second": 454.833,
+      "eval_steps_per_second": 3.581,
+      "step": 50
+    },
+    {
+      "epoch": 0.004044489383215369,
+      "grad_norm": 0.23046875,
+      "learning_rate": 0.001475,
+      "loss": 6.078617095947266,
+      "step": 60
+    },
+    {
+      "epoch": 0.005392652510953826,
+      "grad_norm": 0.1201171875,
+      "learning_rate": 0.001975,
+      "loss": 6.011257553100586,
+      "step": 80
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "grad_norm": 0.53125,
+      "learning_rate": 0.002475,
+      "loss": 5.894095993041992,
+      "step": 100
+    },
+    {
+      "epoch": 0.006740815638692282,
+      "eval_loss": 5.803377628326416,
+      "eval_runtime": 21.0013,
+      "eval_samples_per_second": 453.638,
+      "eval_steps_per_second": 3.571,
+      "step": 100
+    },
+    {
+      "epoch": 0.008088978766430738,
+      "grad_norm": 0.318359375,
+      "learning_rate": 0.0029749999999999998,
+      "loss": 5.779674911499024,
+      "step": 120
+    },
+    {
+      "epoch": 0.009437141894169195,
+      "grad_norm": 0.201171875,
+      "learning_rate": 0.003475,
+      "loss": 5.59539680480957,
+      "step": 140
+    },
+    {
+      "epoch": 0.010111223458038422,
+      "eval_loss": 5.564068794250488,
+      "eval_runtime": 21.0632,
+      "eval_samples_per_second": 452.304,
+      "eval_steps_per_second": 3.561,
+      "step": 150
+    },
+    {
+      "epoch": 0.010785305021907651,
+      "grad_norm": 0.23828125,
+      "learning_rate": 0.003975,
+      "loss": 5.553666305541992,
+      "step": 160
+    },
+    {
+      "epoch": 0.012133468149646108,
+      "grad_norm": 0.3046875,
+      "learning_rate": 0.004475,
+      "loss": 5.471728897094726,
+      "step": 180
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "grad_norm": 0.2109375,
+      "learning_rate": 0.004975,
+      "loss": 5.362456512451172,
+      "step": 200
+    },
+    {
+      "epoch": 0.013481631277384564,
+      "eval_loss": 5.277792930603027,
+      "eval_runtime": 21.0307,
+      "eval_samples_per_second": 453.004,
+      "eval_steps_per_second": 3.566,
+      "step": 200
+    },
+    {
+      "epoch": 0.01482979440512302,
+      "grad_norm": 0.39453125,
+      "learning_rate": 0.005,
+      "loss": 5.181310653686523,
+      "step": 220
+    },
+    {
+      "epoch": 0.016177957532861477,
+      "grad_norm": 0.275390625,
+      "learning_rate": 0.005,
+      "loss": 5.013983917236328,
+      "step": 240
+    },
+    {
+      "epoch": 0.016852039096730706,
+      "eval_loss": 4.766153335571289,
+      "eval_runtime": 20.9959,
+      "eval_samples_per_second": 453.755,
+      "eval_steps_per_second": 3.572,
+      "step": 250
+    },
+    {
+      "epoch": 0.01752612066059993,
+      "grad_norm": 0.7578125,
+      "learning_rate": 0.005,
+      "loss": 4.779726028442383,
+      "step": 260
+    },
+    {
+      "epoch": 0.01887428378833839,
+      "grad_norm": 0.30859375,
+      "learning_rate": 0.005,
+      "loss": 4.661225128173828,
+      "step": 280
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "grad_norm": 0.65625,
+      "learning_rate": 0.005,
+      "loss": 4.492392349243164,
+      "step": 300
+    },
+    {
+      "epoch": 0.020222446916076844,
+      "eval_loss": 4.413177490234375,
+      "eval_runtime": 21.1373,
+      "eval_samples_per_second": 450.72,
+      "eval_steps_per_second": 3.548,
+      "step": 300
+    },
+    {
+      "epoch": 0.021570610043815303,
+      "grad_norm": 0.330078125,
+      "learning_rate": 0.005,
+      "loss": 4.334686660766602,
+      "step": 320
+    },
+    {
+      "epoch": 0.022918773171553757,
+      "grad_norm": 0.8984375,
+      "learning_rate": 0.005,
+      "loss": 4.257295989990235,
+      "step": 340
+    },
+    {
+      "epoch": 0.023592854735422986,
+      "eval_loss": 4.175577640533447,
+      "eval_runtime": 21.1704,
+      "eval_samples_per_second": 450.016,
+      "eval_steps_per_second": 3.543,
+      "step": 350
+    },
+    {
+      "epoch": 0.024266936299292215,
+      "grad_norm": 0.2890625,
+      "learning_rate": 0.005,
+      "loss": 4.192065048217773,
+      "step": 360
+    },
+    {
+      "epoch": 0.02561509942703067,
+      "grad_norm": 0.71875,
+      "learning_rate": 0.005,
+      "loss": 4.055378341674805,
+      "step": 380
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "grad_norm": 0.63671875,
+      "learning_rate": 0.005,
+      "loss": 4.114184951782226,
+      "step": 400
+    },
+    {
+      "epoch": 0.026963262554769128,
+      "eval_loss": 4.11501932144165,
+      "eval_runtime": 21.0092,
+      "eval_samples_per_second": 453.468,
+      "eval_steps_per_second": 3.57,
+      "step": 400
+    },
+    {
+      "epoch": 0.028311425682507583,
+      "grad_norm": 0.322265625,
+      "learning_rate": 0.005,
+      "loss": 4.029239654541016,
+      "step": 420
+    },
+    {
+      "epoch": 0.02965958881024604,
+      "grad_norm": 0.2412109375,
+      "learning_rate": 0.005,
+      "loss": 3.9297679901123046,
+      "step": 440
+    },
+    {
+      "epoch": 0.030333670374115267,
+      "eval_loss": 3.826937675476074,
+      "eval_runtime": 20.9765,
+      "eval_samples_per_second": 454.175,
+      "eval_steps_per_second": 3.575,
+      "step": 450
+    },
+    {
+      "epoch": 0.031007751937984496,
+      "grad_norm": 0.248046875,
+      "learning_rate": 0.005,
+      "loss": 3.8271865844726562,
+      "step": 460
+    },
+    {
+      "epoch": 0.032355915065722954,
+      "grad_norm": 0.251953125,
+      "learning_rate": 0.005,
+      "loss": 3.766727828979492,
+      "step": 480
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "grad_norm": 0.41796875,
+      "learning_rate": 0.005,
+      "loss": 3.703250503540039,
+      "step": 500
+    },
+    {
+      "epoch": 0.03370407819346141,
+      "eval_loss": 3.6817057132720947,
+      "eval_runtime": 21.0653,
+      "eval_samples_per_second": 452.261,
+      "eval_steps_per_second": 3.56,
+      "step": 500
+    },
+    {
+      "epoch": 0.03505224132119986,
+      "grad_norm": 0.30078125,
+      "learning_rate": 0.005,
+      "loss": 3.661212921142578,
+      "step": 520
+    },
+    {
+      "epoch": 0.03640040444893832,
+      "grad_norm": 0.3203125,
+      "learning_rate": 0.005,
+      "loss": 3.599516677856445,
+      "step": 540
+    },
+    {
+      "epoch": 0.03707448601280755,
+      "eval_loss": 3.577031135559082,
+      "eval_runtime": 21.0122,
+      "eval_samples_per_second": 453.404,
+      "eval_steps_per_second": 3.569,
+      "step": 550
+    },
+    {
+      "epoch": 0.03774856757667678,
+      "grad_norm": 0.38671875,
+      "learning_rate": 0.005,
+      "loss": 3.5561344146728517,
+      "step": 560
+    },
+    {
+      "epoch": 0.03909673070441524,
+      "grad_norm": 0.71484375,
+      "learning_rate": 0.005,
+      "loss": 3.6597225189208986,
+      "step": 580
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "grad_norm": 0.27734375,
+      "learning_rate": 0.005,
+      "loss": 3.642597961425781,
+      "step": 600
+    },
+    {
+      "epoch": 0.04044489383215369,
+      "eval_loss": 3.569218635559082,
+      "eval_runtime": 21.0663,
+      "eval_samples_per_second": 452.239,
+      "eval_steps_per_second": 3.56,
+      "step": 600
+    },
+    {
+      "epoch": 0.04179305695989215,
+      "grad_norm": 0.28125,
+      "learning_rate": 0.005,
+      "loss": 3.522428512573242,
+      "step": 620
+    },
+    {
+      "epoch": 0.043141220087630605,
+      "grad_norm": 0.5,
+      "learning_rate": 0.005,
+      "loss": 3.4951053619384767,
+      "step": 640
+    },
+    {
+      "epoch": 0.043815301651499834,
+      "eval_loss": 3.4496421813964844,
+      "eval_runtime": 21.0478,
+      "eval_samples_per_second": 452.637,
+      "eval_steps_per_second": 3.563,
+      "step": 650
+    },
+    {
+      "epoch": 0.044489383215369056,
+      "grad_norm": 0.423828125,
+      "learning_rate": 0.005,
+      "loss": 3.4415531158447266,
+      "step": 660
+    },
+    {
+      "epoch": 0.045837546343107514,
+      "grad_norm": 0.28125,
+      "learning_rate": 0.005,
+      "loss": 3.430613708496094,
+      "step": 680
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "grad_norm": 0.296875,
+      "learning_rate": 0.005,
+      "loss": 3.3803379058837892,
+      "step": 700
+    },
+    {
+      "epoch": 0.04718570947084597,
+      "eval_loss": 3.3822414875030518,
+      "eval_runtime": 21.146,
+      "eval_samples_per_second": 450.535,
+      "eval_steps_per_second": 3.547,
+      "step": 700
+    },
+    {
+      "epoch": 0.04853387259858443,
+      "grad_norm": 0.451171875,
+      "learning_rate": 0.005,
+      "loss": 3.3574844360351563,
+      "step": 720
+    },
+    {
+      "epoch": 0.04988203572632288,
+      "grad_norm": 0.314453125,
+      "learning_rate": 0.005,
+      "loss": 3.3271942138671875,
+      "step": 740
+    },
+    {
+      "epoch": 0.05055611729019211,
+      "eval_loss": 3.326965570449829,
+      "eval_runtime": 21.0998,
+      "eval_samples_per_second": 451.52,
+      "eval_steps_per_second": 3.555,
+      "step": 750
+    },
+    {
+      "epoch": 0.05123019885406134,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.005,
+      "loss": 3.3225662231445314,
+      "step": 760
+    },
+    {
+      "epoch": 0.0525783619817998,
+      "grad_norm": 0.328125,
+      "learning_rate": 0.005,
+      "loss": 3.3127796173095705,
+      "step": 780
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "grad_norm": 0.31640625,
+      "learning_rate": 0.005,
+      "loss": 3.2805011749267576,
+      "step": 800
+    },
+    {
+      "epoch": 0.053926525109538256,
+      "eval_loss": 3.279794454574585,
+      "eval_runtime": 21.1218,
+      "eval_samples_per_second": 451.051,
+      "eval_steps_per_second": 3.551,
+      "step": 800
+    },
+    {
+      "epoch": 0.05527468823727671,
+      "grad_norm": 0.76171875,
+      "learning_rate": 0.005,
+      "loss": 3.288113021850586,
+      "step": 820
+    },
+    {
+      "epoch": 0.056622851365015166,
+      "grad_norm": 0.2890625,
+      "learning_rate": 0.005,
+      "loss": 3.2744274139404297,
+      "step": 840
+    },
+    {
+      "epoch": 0.057296932928884395,
+      "eval_loss": 3.242783784866333,
+      "eval_runtime": 21.0246,
+      "eval_samples_per_second": 453.135,
+      "eval_steps_per_second": 3.567,
+      "step": 850
+    },
+    {
+      "epoch": 0.057971014492753624,
+      "grad_norm": 0.30078125,
+      "learning_rate": 0.005,
+      "loss": 3.2257854461669924,
+      "step": 860
+    },
+    {
+      "epoch": 0.05931917762049208,
+      "grad_norm": 0.7734375,
+      "learning_rate": 0.005,
+      "loss": 3.2115924835205076,
+      "step": 880
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "grad_norm": 0.4453125,
+      "learning_rate": 0.005,
+      "loss": 3.215483856201172,
+      "step": 900
+    },
+    {
+      "epoch": 0.06066734074823053,
+      "eval_loss": 3.204148292541504,
+      "eval_runtime": 21.0862,
+      "eval_samples_per_second": 451.812,
+      "eval_steps_per_second": 3.557,
+      "step": 900
+    },
+    {
+      "epoch": 0.06201550387596899,
+      "grad_norm": 0.4296875,
+      "learning_rate": 0.005,
+      "loss": 3.200112533569336,
+      "step": 920
+    },
+    {
+      "epoch": 0.06336366700370745,
+      "grad_norm": 0.26953125,
+      "learning_rate": 0.005,
+      "loss": 3.1862085342407225,
+      "step": 940
+    },
+    {
+      "epoch": 0.06403774856757667,
+      "eval_loss": 3.176638126373291,
+      "eval_runtime": 21.3796,
+      "eval_samples_per_second": 445.612,
+      "eval_steps_per_second": 3.508,
+      "step": 950
+    },
+    {
+      "epoch": 0.06471183013144591,
+      "grad_norm": 0.400390625,
+      "learning_rate": 0.005,
+      "loss": 3.1680328369140627,
+      "step": 960
+    },
+    {
+      "epoch": 0.06605999325918437,
+      "grad_norm": 0.494140625,
+      "learning_rate": 0.005,
+      "loss": 3.1643564224243166,
+      "step": 980
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "grad_norm": 0.60546875,
+      "learning_rate": 0.005,
+      "loss": 3.1570945739746095,
+      "step": 1000
+    },
+    {
+      "epoch": 0.06740815638692282,
+      "eval_loss": 3.1693739891052246,
+      "eval_runtime": 20.9569,
+      "eval_samples_per_second": 454.599,
+      "eval_steps_per_second": 3.579,
+      "step": 1000
+    }
+  ],
+  "logging_steps": 20,
+  "max_steps": 1000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4839413121024000.0,
+  "train_batch_size": 64,
+  "trial_name": null,
+  "trial_params": null
+}

zain/Activation/out/glu-silu-150L_run/checkpoint-1000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dbc5a45fe51c91ab1923ef9841ab866304e4eb043fc0626e0bd4311795fca98
+size 4920

zain/Activation/out/glu-silu-150L_run/config.json

@@ -0,0 +1,36 @@
+{
+  "activation": "silu",
+  "architectures": [
+    "TinyLlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 2,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 128,
+  "initializer_range": 0.02,
+  "intermediate_size": 256,
+  "max_position_embeddings": 512,
+  "mlp_bias": false,
+  "mlp_type": "glu",
+  "model_type": "tiny_llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 150,
+  "num_key_value_heads": 4,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": true,
+  "tokenizer_name": "w-ahmad/tiny-stories-tokenizer",
+  "transformers_version": "5.16.0.dev0",
+  "use_cache": false,
+  "vocab_size": 4096,
+  "waleed_beta": 10.0
+}

zain/Activation/out/glu-silu-150L_run/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c19d4f22bca1d6c50b4226f59e84e12d53357840e7cb70cf90784aefb45a5314
+size 50427120

zain/Activation/out/glu-silu-150L_run/tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

zain/Activation/out/glu-silu-150L_run/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dbc5a45fe51c91ab1923ef9841ab866304e4eb043fc0626e0bd4311795fca98
+size 4920

zain/Activation/out/glu-silu-150L_run/training_log.jsonl

@@ -0,0 +1,72 @@
+{"step": 20, "epoch": 0.0013481631277384564, "timestamp": 1786557193.6174574, "loss": 7.965912628173828, "grad_norm": 1.2578125, "learning_rate": 0.000475, "train/total_time_seconds": 7.7677748911082745, "train/time_per_step_avg": 0.3883887445554137, "train/epoch_time_elapsed": 8.869889877736568, "train/estimated_remaining_minutes": 6.343682827738424}
+{"step": 40, "epoch": 0.002696326255476913, "timestamp": 1786557201.5156848, "loss": 6.833769226074219, "grad_norm": 0.875, "learning_rate": 0.0009750000000000001, "train/total_time_seconds": 14.692789886146784, "train/time_per_step_avg": 0.3673197471536696, "train/epoch_time_elapsed": 16.76811734586954, "train/estimated_remaining_minutes": 5.877115954458714}
+{"step": 50, "epoch": 0.003370407819346141, "timestamp": 1786557226.4327006, "eval_loss": 6.036951541900635, "eval_runtime": 20.9461, "eval_samples_per_second": 454.833, "eval_steps_per_second": 3.581, "train/total_time_seconds": 18.16822624206543, "train/time_per_step_avg": 0.3633645248413086, "train/epoch_time_elapsed": 41.68513401597738, "train/estimated_remaining_minutes": 5.753271643320719}
+{"step": 60, "epoch": 0.004044489383215369, "timestamp": 1786557230.3814466, "loss": 6.078617095947266, "grad_norm": 0.23046875, "learning_rate": 0.001475, "train/total_time_seconds": 21.630605444312096, "train/time_per_step_avg": 0.3605100907385349, "train/epoch_time_elapsed": 45.63387914374471, "train/estimated_remaining_minutes": 5.647991421570381}
+{"step": 80, "epoch": 0.005392652510953826, "timestamp": 1786557238.3316472, "loss": 6.011257553100586, "grad_norm": 0.1201171875, "learning_rate": 0.001975, "train/total_time_seconds": 28.592904217541218, "train/time_per_step_avg": 0.35741130271926524, "train/epoch_time_elapsed": 53.58407938852906, "train/estimated_remaining_minutes": 5.4803066416954005}
+{"step": 100, "epoch": 0.006740815638692282, "timestamp": 1786557246.2685323, "loss": 5.894095993041992, "grad_norm": 0.53125, "learning_rate": 0.002475, "train/total_time_seconds": 35.53966587409377, "train/time_per_step_avg": 0.35539665874093773, "train/epoch_time_elapsed": 61.52096566557884, "train/estimated_remaining_minutes": 5.330949881114066}
+{"step": 100, "epoch": 0.006740815638692282, "timestamp": 1786557267.2714484, "eval_loss": 5.803377628326416, "eval_runtime": 21.0013, "eval_samples_per_second": 453.638, "eval_steps_per_second": 3.571, "train/total_time_seconds": 35.53966587409377, "train/time_per_step_avg": 0.35539665874093773, "train/epoch_time_elapsed": 82.52388128265738, "train/estimated_remaining_minutes": 5.330949881114066}
+{"step": 120, "epoch": 0.008088978766430738, "timestamp": 1786557275.2251031, "loss": 5.779674911499024, "grad_norm": 0.318359375, "learning_rate": 0.0029749999999999998, "train/total_time_seconds": 42.507759012281895, "train/time_per_step_avg": 0.3473998412117362, "train/epoch_time_elapsed": 90.47753590345383, "train/estimated_remaining_minutes": 5.195392768167787}
+{"step": 140, "epoch": 0.009437141894169195, "timestamp": 1786557283.4390974, "loss": 5.59539680480957, "grad_norm": 0.201171875, "learning_rate": 0.003475, "train/total_time_seconds": 49.740279983729124, "train/time_per_step_avg": 0.3504749009758234, "train/epoch_time_elapsed": 98.6915314309299, "train/estimated_remaining_minutes": 5.09245723642941}
+{"step": 150, "epoch": 0.010111223458038422, "timestamp": 1786557308.5183148, "eval_loss": 5.564068794250488, "eval_runtime": 21.0632, "eval_samples_per_second": 452.304, "eval_steps_per_second": 3.561, "train/total_time_seconds": 53.26917500048876, "train/time_per_step_avg": 0.3510094875842333, "train/epoch_time_elapsed": 123.77074796706438, "train/estimated_remaining_minutes": 5.03097763893505}
+{"step": 160, "epoch": 0.010785305021907651, "timestamp": 1786557312.4706361, "loss": 5.553666305541992, "grad_norm": 0.23828125, "learning_rate": 0.003975, "train/total_time_seconds": 56.731630615890026, "train/time_per_step_avg": 0.3510102517157793, "train/epoch_time_elapsed": 127.72306951135397, "train/estimated_remaining_minutes": 4.964017678890377}
+{"step": 180, "epoch": 0.012133468149646108, "timestamp": 1786557320.358641, "loss": 5.471728897094726, "grad_norm": 0.3046875, "learning_rate": 0.004475, "train/total_time_seconds": 63.65416628494859, "train/time_per_step_avg": 0.35061262067407367, "train/epoch_time_elapsed": 135.61107504740357, "train/estimated_remaining_minutes": 4.833001514227577}
+{"step": 200, "epoch": 0.013481631277384564, "timestamp": 1786557328.2829368, "loss": 5.362456512451172, "grad_norm": 0.2109375, "learning_rate": 0.004975, "train/total_time_seconds": 70.61671846732497, "train/time_per_step_avg": 0.350770525932312, "train/epoch_time_elapsed": 143.53537099063396, "train/estimated_remaining_minutes": 4.707781231154998}
+{"step": 200, "epoch": 0.013481631277384564, "timestamp": 1786557349.3152652, "eval_loss": 5.277792930603027, "eval_runtime": 21.0307, "eval_samples_per_second": 453.004, "eval_steps_per_second": 3.566, "train/total_time_seconds": 70.61671846732497, "train/time_per_step_avg": 0.350770525932312, "train/epoch_time_elapsed": 164.56769954413176, "train/estimated_remaining_minutes": 4.707781231154998}
+{"step": 220, "epoch": 0.01482979440512302, "timestamp": 1786557357.3834257, "loss": 5.181310653686523, "grad_norm": 0.39453125, "learning_rate": 0.005, "train/total_time_seconds": 77.70512066781521, "train/time_per_step_avg": 0.35197361655533316, "train/epoch_time_elapsed": 172.6358605362475, "train/estimated_remaining_minutes": 4.59166622127999}
+{"step": 240, "epoch": 0.016177957532861477, "timestamp": 1786557365.4349813, "loss": 5.013983917236328, "grad_norm": 0.275390625, "learning_rate": 0.005, "train/total_time_seconds": 84.76927033439279, "train/time_per_step_avg": 0.35028990350663664, "train/epoch_time_elapsed": 180.68741608783603, "train/estimated_remaining_minutes": 4.473933712092953}
+{"step": 250, "epoch": 0.016852039096730706, "timestamp": 1786557390.490584, "eval_loss": 4.766153335571289, "eval_runtime": 20.9959, "eval_samples_per_second": 453.755, "eval_steps_per_second": 3.572, "train/total_time_seconds": 88.30736941099167, "train/time_per_step_avg": 0.3503819441050291, "train/epoch_time_elapsed": 205.74301797151566, "train/estimated_remaining_minutes": 4.415368470549583}
+{"step": 260, "epoch": 0.01752612066059993, "timestamp": 1786557394.7535915, "loss": 4.779726028442383, "grad_norm": 0.7578125, "learning_rate": 0.005, "train/total_time_seconds": 92.07968695834279, "train/time_per_step_avg": 0.35348056342452766, "train/epoch_time_elapsed": 210.00602655112743, "train/estimated_remaining_minutes": 4.367882586485491}
+{"step": 280, "epoch": 0.01887428378833839, "timestamp": 1786557402.8140562, "loss": 4.661225128173828, "grad_norm": 0.30859375, "learning_rate": 0.005, "train/total_time_seconds": 99.15521595999599, "train/time_per_step_avg": 0.35501049675047397, "train/epoch_time_elapsed": 218.06649118289351, "train/estimated_remaining_minutes": 4.249509255428399}
+{"step": 300, "epoch": 0.020222446916076844, "timestamp": 1786557410.8971837, "loss": 4.492392349243164, "grad_norm": 0.65625, "learning_rate": 0.005, "train/total_time_seconds": 106.24776089936495, "train/time_per_step_avg": 0.3563104243203998, "train/epoch_time_elapsed": 226.14961894974113, "train/estimated_remaining_minutes": 4.131857368308637}
+{"step": 300, "epoch": 0.020222446916076844, "timestamp": 1786557432.0359025, "eval_loss": 4.413177490234375, "eval_runtime": 21.1373, "eval_samples_per_second": 450.72, "eval_steps_per_second": 3.548, "train/total_time_seconds": 106.24776089936495, "train/time_per_step_avg": 0.3563104243203998, "train/epoch_time_elapsed": 247.2883375324309, "train/estimated_remaining_minutes": 4.131857368308637}
+{"step": 320, "epoch": 0.021570610043815303, "timestamp": 1786557440.0632358, "loss": 4.334686660766602, "grad_norm": 0.330078125, "learning_rate": 0.005, "train/total_time_seconds": 113.29175987839699, "train/time_per_step_avg": 0.3558663921058178, "train/epoch_time_elapsed": 255.31567088514566, "train/estimated_remaining_minutes": 4.012416495693227}
+{"step": 340, "epoch": 0.022918773171553757, "timestamp": 1786557448.0978491, "loss": 4.257295989990235, "grad_norm": 0.8984375, "learning_rate": 0.005, "train/total_time_seconds": 120.34191355109215, "train/time_per_step_avg": 0.3557264321669936, "train/epoch_time_elapsed": 263.3502836711705, "train/estimated_remaining_minutes": 3.8934148501823933}
+{"step": 350, "epoch": 0.023592854735422986, "timestamp": 1786557473.2799585, "eval_loss": 4.175577640533447, "eval_runtime": 21.1704, "eval_samples_per_second": 450.016, "eval_steps_per_second": 3.543, "train/total_time_seconds": 123.86411083117127, "train/time_per_step_avg": 0.35556741420179605, "train/epoch_time_elapsed": 288.5323930978775, "train/estimated_remaining_minutes": 3.8338891447743486}
+{"step": 360, "epoch": 0.024266936299292215, "timestamp": 1786557477.329923, "loss": 4.192065048217773, "grad_norm": 0.2890625, "learning_rate": 0.005, "train/total_time_seconds": 127.41973203793168, "train/time_per_step_avg": 0.3534004507958889, "train/epoch_time_elapsed": 292.58235804736614, "train/estimated_remaining_minutes": 3.7753994677905687}
+{"step": 380, "epoch": 0.02561509942703067, "timestamp": 1786557485.3714654, "loss": 4.055378341674805, "grad_norm": 0.71875, "learning_rate": 0.005, "train/total_time_seconds": 134.47621612250805, "train/time_per_step_avg": 0.3532100016251206, "train/epoch_time_elapsed": 300.6239004135132, "train/estimated_remaining_minutes": 3.6568093857874997}
+{"step": 400, "epoch": 0.026963262554769128, "timestamp": 1786557493.636982, "loss": 4.114184951782226, "grad_norm": 0.63671875, "learning_rate": 0.005, "train/total_time_seconds": 141.75298693031073, "train/time_per_step_avg": 0.3550522603094578, "train/epoch_time_elapsed": 308.8894155956805, "train/estimated_remaining_minutes": 3.543824673257768}
+{"step": 400, "epoch": 0.026963262554769128, "timestamp": 1786557514.6479268, "eval_loss": 4.11501932144165, "eval_runtime": 21.0092, "eval_samples_per_second": 453.468, "eval_steps_per_second": 3.57, "train/total_time_seconds": 141.75298693031073, "train/time_per_step_avg": 0.3550522603094578, "train/epoch_time_elapsed": 329.900360442698, "train/estimated_remaining_minutes": 3.543824673257768}
+{"step": 420, "epoch": 0.028311425682507583, "timestamp": 1786557522.5884686, "loss": 4.029239654541016, "grad_norm": 0.322265625, "learning_rate": 0.005, "train/total_time_seconds": 148.7272277995944, "train/time_per_step_avg": 0.35435467921197417, "train/epoch_time_elapsed": 337.8409028239548, "train/estimated_remaining_minutes": 3.423086989038284}
+{"step": 440, "epoch": 0.02965958881024604, "timestamp": 1786557530.4966893, "loss": 3.9297679901123046, "grad_norm": 0.2412109375, "learning_rate": 0.005, "train/total_time_seconds": 155.6698176935315, "train/time_per_step_avg": 0.35327904142439365, "train/epoch_time_elapsed": 345.74912352859974, "train/estimated_remaining_minutes": 3.302087041984002}
+{"step": 450, "epoch": 0.030333670374115267, "timestamp": 1786557555.4244235, "eval_loss": 3.826937675476074, "eval_runtime": 20.9765, "eval_samples_per_second": 454.175, "eval_steps_per_second": 3.575, "train/total_time_seconds": 159.13453048840165, "train/time_per_step_avg": 0.3527041965723038, "train/epoch_time_elapsed": 370.6768567673862, "train/estimated_remaining_minutes": 3.2416293247637373}
+{"step": 460, "epoch": 0.031007751937984496, "timestamp": 1786557559.4379292, "loss": 3.8271865844726562, "grad_norm": 0.248046875, "learning_rate": 0.005, "train/total_time_seconds": 162.65948490053415, "train/time_per_step_avg": 0.35239752862602475, "train/epoch_time_elapsed": 374.69036443531513, "train/estimated_remaining_minutes": 3.182468182836538}
+{"step": 480, "epoch": 0.032355915065722954, "timestamp": 1786557567.5498652, "loss": 3.766727828979492, "grad_norm": 0.251953125, "learning_rate": 0.005, "train/total_time_seconds": 169.77665081992745, "train/time_per_step_avg": 0.353004346974194, "train/epoch_time_elapsed": 382.80230047926307, "train/estimated_remaining_minutes": 3.065411750915357}
+{"step": 500, "epoch": 0.03370407819346141, "timestamp": 1786557575.617877, "loss": 3.703250503540039, "grad_norm": 0.41796875, "learning_rate": 0.005, "train/total_time_seconds": 176.84840467572212, "train/time_per_step_avg": 0.35095417745411395, "train/epoch_time_elapsed": 390.87031165137887, "train/estimated_remaining_minutes": 2.9474734112620355}
+{"step": 500, "epoch": 0.03370407819346141, "timestamp": 1786557596.684591, "eval_loss": 3.6817057132720947, "eval_runtime": 21.0653, "eval_samples_per_second": 452.261, "eval_steps_per_second": 3.56, "train/total_time_seconds": 176.84840467572212, "train/time_per_step_avg": 0.35095417745411395, "train/epoch_time_elapsed": 411.9370235167444, "train/estimated_remaining_minutes": 2.9474734112620355}
+{"step": 520, "epoch": 0.03505224132119986, "timestamp": 1786557604.6020725, "loss": 3.661212921142578, "grad_norm": 0.30078125, "learning_rate": 0.005, "train/total_time_seconds": 183.79093367606401, "train/time_per_step_avg": 0.35063705876469614, "train/epoch_time_elapsed": 419.85450745746493, "train/estimated_remaining_minutes": 2.8275528257856}
+{"step": 540, "epoch": 0.03640040444893832, "timestamp": 1786557612.8317752, "loss": 3.599516677856445, "grad_norm": 0.3203125, "learning_rate": 0.005, "train/total_time_seconds": 191.0475917197764, "train/time_per_step_avg": 0.3537777402624488, "train/epoch_time_elapsed": 428.0842099711299, "train/estimated_remaining_minutes": 2.7124040799721336}
+{"step": 550, "epoch": 0.03707448601280755, "timestamp": 1786557637.8027244, "eval_loss": 3.577031135559082, "eval_runtime": 21.0122, "eval_samples_per_second": 453.404, "eval_steps_per_second": 3.569, "train/total_time_seconds": 194.519395314157, "train/time_per_step_avg": 0.3538486482575536, "train/epoch_time_elapsed": 453.05515737086535, "train/estimated_remaining_minutes": 2.6525372088294135}
+{"step": 560, "epoch": 0.03774856757667678, "timestamp": 1786557641.7856886, "loss": 3.5561344146728517, "grad_norm": 0.38671875, "learning_rate": 0.005, "train/total_time_seconds": 197.99239484220743, "train/time_per_step_avg": 0.3533290994167328, "train/epoch_time_elapsed": 457.03812321648, "train/estimated_remaining_minutes": 2.5927575515050973}
+{"step": 580, "epoch": 0.03909673070441524, "timestamp": 1786557649.796253, "loss": 3.6597225189208986, "grad_norm": 0.71484375, "learning_rate": 0.005, "train/total_time_seconds": 204.98366295918822, "train/time_per_step_avg": 0.3520701213926077, "train/epoch_time_elapsed": 465.04868764802814, "train/estimated_remaining_minutes": 2.4739407598522716}
+{"step": 600, "epoch": 0.04044489383215369, "timestamp": 1786557657.7552896, "loss": 3.642597961425781, "grad_norm": 0.27734375, "learning_rate": 0.005, "train/total_time_seconds": 211.96490164473653, "train/time_per_step_avg": 0.35116496969014405, "train/epoch_time_elapsed": 473.0077244900167, "train/estimated_remaining_minutes": 2.355165573830406}
+{"step": 600, "epoch": 0.04044489383215369, "timestamp": 1786557678.8231282, "eval_loss": 3.569218635559082, "eval_runtime": 21.0663, "eval_samples_per_second": 452.239, "eval_steps_per_second": 3.56, "train/total_time_seconds": 211.96490164473653, "train/time_per_step_avg": 0.35116496969014405, "train/epoch_time_elapsed": 494.07556308060884, "train/estimated_remaining_minutes": 2.355165573830406}
+{"step": 620, "epoch": 0.04179305695989215, "timestamp": 1786557686.9847145, "loss": 3.522428512573242, "grad_norm": 0.28125, "learning_rate": 0.005, "train/total_time_seconds": 219.13101323321462, "train/time_per_step_avg": 0.353400795571506, "train/epoch_time_elapsed": 502.23714938387275, "train/estimated_remaining_minutes": 2.2384350814145577}
+{"step": 640, "epoch": 0.043141220087630605, "timestamp": 1786557695.0759084, "loss": 3.4951053619384767, "grad_norm": 0.5, "learning_rate": 0.005, "train/total_time_seconds": 226.23501834645867, "train/time_per_step_avg": 0.35187426626682283, "train/epoch_time_elapsed": 510.3283435218036, "train/estimated_remaining_minutes": 2.1209532969980502}
+{"step": 650, "epoch": 0.043815301651499834, "timestamp": 1786557720.1554163, "eval_loss": 3.4496421813964844, "eval_runtime": 21.0478, "eval_samples_per_second": 452.637, "eval_steps_per_second": 3.563, "train/total_time_seconds": 229.77535781264305, "train/time_per_step_avg": 0.35255962498486043, "train/epoch_time_elapsed": 535.4078501574695, "train/estimated_remaining_minutes": 2.0620865444724377}
+{"step": 660, "epoch": 0.044489383215369056, "timestamp": 1786557724.1980085, "loss": 3.4415531158447266, "grad_norm": 0.423828125, "learning_rate": 0.005, "train/total_time_seconds": 233.32044852897525, "train/time_per_step_avg": 0.3532805368676782, "train/epoch_time_elapsed": 539.4504432417452, "train/estimated_remaining_minutes": 2.003256376258878}
+{"step": 680, "epoch": 0.045837546343107514, "timestamp": 1786557732.4899755, "loss": 3.430613708496094, "grad_norm": 0.28125, "learning_rate": 0.005, "train/total_time_seconds": 240.63763910531998, "train/time_per_step_avg": 0.35653976146131755, "train/epoch_time_elapsed": 547.7424106113613, "train/estimated_remaining_minutes": 1.887354032198588}
+{"step": 700, "epoch": 0.04718570947084597, "timestamp": 1786557740.6322117, "loss": 3.3803379058837892, "grad_norm": 0.296875, "learning_rate": 0.005, "train/total_time_seconds": 247.7928595095873, "train/time_per_step_avg": 0.35827957864850757, "train/epoch_time_elapsed": 555.8846466131508, "train/estimated_remaining_minutes": 1.7699489964970523}
+{"step": 700, "epoch": 0.04718570947084597, "timestamp": 1786557761.7795937, "eval_loss": 3.3822414875030518, "eval_runtime": 21.146, "eval_samples_per_second": 450.535, "eval_steps_per_second": 3.547, "train/total_time_seconds": 247.7928595095873, "train/time_per_step_avg": 0.35827957864850757, "train/epoch_time_elapsed": 577.0320283398032, "train/estimated_remaining_minutes": 1.7699489964970523}
+{"step": 720, "epoch": 0.04853387259858443, "timestamp": 1786557769.8749285, "loss": 3.3574844360351563, "grad_norm": 0.451171875, "learning_rate": 0.005, "train/total_time_seconds": 254.90526371449232, "train/time_per_step_avg": 0.35774250481277703, "train/epoch_time_elapsed": 585.127363409847, "train/estimated_remaining_minutes": 1.6521637462976353}
+{"step": 740, "epoch": 0.04988203572632288, "timestamp": 1786557778.1536245, "loss": 3.3271942138671875, "grad_norm": 0.314453125, "learning_rate": 0.005, "train/total_time_seconds": 262.1181674040854, "train/time_per_step_avg": 0.35883149057626723, "train/epoch_time_elapsed": 593.4060596898198, "train/estimated_remaining_minutes": 1.534926205519419}
+{"step": 750, "epoch": 0.05055611729019211, "timestamp": 1786557803.2730293, "eval_loss": 3.326965570449829, "eval_runtime": 21.0998, "eval_samples_per_second": 451.52, "eval_steps_per_second": 3.555, "train/total_time_seconds": 265.6501111611724, "train/time_per_step_avg": 0.35874753348529337, "train/epoch_time_elapsed": 618.5254634954035, "train/estimated_remaining_minutes": 1.4758339508954024}
+{"step": 760, "epoch": 0.05123019885406134, "timestamp": 1786557807.5223897, "loss": 3.3225662231445314, "grad_norm": 0.337890625, "learning_rate": 0.005, "train/total_time_seconds": 269.36594320461154, "train/time_per_step_avg": 0.3604549467563629, "train/epoch_time_elapsed": 622.774824872613, "train/estimated_remaining_minutes": 1.417715490550587}
+{"step": 780, "epoch": 0.0525783619817998, "timestamp": 1786557815.562599, "loss": 3.3127796173095705, "grad_norm": 0.328125, "learning_rate": 0.005, "train/total_time_seconds": 276.4333055205643, "train/time_per_step_avg": 0.3579566641524434, "train/epoch_time_elapsed": 630.815034121275, "train/estimated_remaining_minutes": 1.2994728037291485}
+{"step": 800, "epoch": 0.053926525109538256, "timestamp": 1786557823.6051483, "loss": 3.2805011749267576, "grad_norm": 0.31640625, "learning_rate": 0.005, "train/total_time_seconds": 283.50869573652744, "train/time_per_step_avg": 0.35715836226940156, "train/epoch_time_elapsed": 638.8575837723911, "train/estimated_remaining_minutes": 1.181286232235531}
+{"step": 800, "epoch": 0.053926525109538256, "timestamp": 1786557844.7283092, "eval_loss": 3.279794454574585, "eval_runtime": 21.1218, "eval_samples_per_second": 451.051, "eval_steps_per_second": 3.551, "train/total_time_seconds": 283.50869573652744, "train/time_per_step_avg": 0.35715836226940156, "train/epoch_time_elapsed": 659.9807434231043, "train/estimated_remaining_minutes": 1.181286232235531}
+{"step": 820, "epoch": 0.05527468823727671, "timestamp": 1786557852.9730716, "loss": 3.288113021850586, "grad_norm": 0.76171875, "learning_rate": 0.005, "train/total_time_seconds": 290.78088065981865, "train/time_per_step_avg": 0.3587561694532633, "train/epoch_time_elapsed": 668.2255069240928, "train/estimated_remaining_minutes": 1.0638324902188487}
+{"step": 840, "epoch": 0.056622851365015166, "timestamp": 1786557861.1435368, "loss": 3.2744274139404297, "grad_norm": 0.2890625, "learning_rate": 0.005, "train/total_time_seconds": 297.97745206579566, "train/time_per_step_avg": 0.3585928466171026, "train/epoch_time_elapsed": 676.3959720656276, "train/estimated_remaining_minutes": 0.9459601652882401}
+{"step": 850, "epoch": 0.057296932928884395, "timestamp": 1786557886.2209425, "eval_loss": 3.242783784866333, "eval_runtime": 21.0246, "eval_samples_per_second": 453.135, "eval_steps_per_second": 3.567, "train/total_time_seconds": 301.53453793376684, "train/time_per_step_avg": 0.3588442677259445, "train/epoch_time_elapsed": 701.4733771942556, "train/estimated_remaining_minutes": 0.8868662880404906}
+{"step": 860, "epoch": 0.057971014492753624, "timestamp": 1786557890.2724154, "loss": 3.2257854461669924, "grad_norm": 0.30078125, "learning_rate": 0.005, "train/total_time_seconds": 305.0943289361894, "train/time_per_step_avg": 0.3572838573157787, "train/epoch_time_elapsed": 705.5248506069183, "train/estimated_remaining_minutes": 0.8277753110671805}
+{"step": 880, "epoch": 0.05931917762049208, "timestamp": 1786557898.407817, "loss": 3.2115924835205076, "grad_norm": 0.7734375, "learning_rate": 0.005, "train/total_time_seconds": 312.24482537060976, "train/time_per_step_avg": 0.3581151985004544, "train/epoch_time_elapsed": 713.6602519154549, "train/estimated_remaining_minutes": 0.7096473303877494}
+{"step": 900, "epoch": 0.06066734074823053, "timestamp": 1786557906.474679, "loss": 3.215483856201172, "grad_norm": 0.4453125, "learning_rate": 0.005, "train/total_time_seconds": 319.3302355930209, "train/time_per_step_avg": 0.35821539856493473, "train/epoch_time_elapsed": 721.7271142341197, "train/estimated_remaining_minutes": 0.591352288135224}
+{"step": 900, "epoch": 0.06066734074823053, "timestamp": 1786557927.562245, "eval_loss": 3.204148292541504, "eval_runtime": 21.0862, "eval_samples_per_second": 451.812, "eval_steps_per_second": 3.557, "train/total_time_seconds": 319.3302355930209, "train/time_per_step_avg": 0.35821539856493473, "train/epoch_time_elapsed": 742.8146789930761, "train/estimated_remaining_minutes": 0.591352288135224}
+{"step": 920, "epoch": 0.06201550387596899, "timestamp": 1786557935.6161351, "loss": 3.200112533569336, "grad_norm": 0.4296875, "learning_rate": 0.005, "train/total_time_seconds": 326.4064021706581, "train/time_per_step_avg": 0.35625521510839464, "train/epoch_time_elapsed": 750.8685701228678, "train/estimated_remaining_minutes": 0.4730527567690697}
+{"step": 940, "epoch": 0.06336366700370745, "timestamp": 1786557943.8464644, "loss": 3.1862085342407225, "grad_norm": 0.26953125, "learning_rate": 0.005, "train/total_time_seconds": 333.6590396165848, "train/time_per_step_avg": 0.3568158755078912, "train/epoch_time_elapsed": 759.0988992676139, "train/estimated_remaining_minutes": 0.35495642512402636}
+{"step": 950, "epoch": 0.06403774856757667, "timestamp": 1786557969.2524087, "eval_loss": 3.176638126373291, "eval_runtime": 21.3796, "eval_samples_per_second": 445.612, "eval_steps_per_second": 3.508, "train/total_time_seconds": 337.19563449174166, "train/time_per_step_avg": 0.35661096557974814, "train/epoch_time_elapsed": 784.5048425756395, "train/estimated_remaining_minutes": 0.29578564429100146}
+{"step": 960, "epoch": 0.06471183013144591, "timestamp": 1786557973.2440467, "loss": 3.1680328369140627, "grad_norm": 0.400390625, "learning_rate": 0.005, "train/total_time_seconds": 340.68483125418425, "train/time_per_step_avg": 0.3559050231799483, "train/epoch_time_elapsed": 788.496480870992, "train/estimated_remaining_minutes": 0.23658668837096125}
+{"step": 980, "epoch": 0.06605999325918437, "timestamp": 1786557981.187124, "loss": 3.1643564224243166, "grad_norm": 0.494140625, "learning_rate": 0.005, "train/total_time_seconds": 347.66380301490426, "train/time_per_step_avg": 0.354189776442945, "train/epoch_time_elapsed": 796.4395584575832, "train/estimated_remaining_minutes": 0.11825299422275656}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786557989.1446016, "loss": 3.1570945739746095, "grad_norm": 0.60546875, "learning_rate": 0.005, "train/total_time_seconds": 354.6520741917193, "train/time_per_step_avg": 0.3532183859869838, "train/epoch_time_elapsed": 804.3970361538231, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786558010.1032054, "eval_loss": 3.1693739891052246, "eval_runtime": 20.9569, "eval_samples_per_second": 454.599, "eval_steps_per_second": 3.579, "train/total_time_seconds": 354.6520741917193, "train/time_per_step_avg": 0.3532183859869838, "train/epoch_time_elapsed": 825.3556394092739, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786558010.52342, "train_runtime": 826.5493, "train_samples_per_second": 77.43, "train_steps_per_second": 1.21, "total_flos": 4839413121024000.0, "train_loss": 4.179319068908692, "train/total_time_seconds": 354.6520741917193, "train/time_per_step_avg": 0.3532183859869838, "train/epoch_time_elapsed": 825.7758530080318, "train/estimated_remaining_minutes": 0.0}
+{"step": 1000, "epoch": 0.06740815638692282, "timestamp": 1786558031.5403202, "eval_loss": 3.1693739891052246, "eval_runtime": 21.013, "eval_samples_per_second": 453.385, "eval_steps_per_second": 3.569, "train/total_time_seconds": 354.6520741917193, "train/time_per_step_avg": 0.3532183859869838, "train/epoch_time_elapsed": 846.7927529215813, "train/estimated_remaining_minutes": 0.0}

zain/Activation/sweep.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""Sweep explicit GLU / MLP variants with identical data and hyperparameters."""
+import argparse
+import copy
+import json
+import re
+import time
+import os
+from pathlib import Path
+
+import yaml
+import wandb
+import torch
+from transformers import AutoTokenizer, set_seed
+from exp import TinyLlamaConfig, TinyLlamaForCausalLM, build_dataset, create_trainer
+
+
+def format_param_count(total_params: int) -> str:
+    """Return human‑readable string with M or B suffix, 1 decimal."""
+    if total_params >= 1e9:
+        return f"{total_params / 1e9:.1f}B"
+    else:
+        return f"{total_params / 1e6:.1f}M"
+
+
+def parse_variant(variant: str):
+    """
+    Parse variant string into (prefix, activation, layers).
+    Supports:
+        glu-silu-waleed10
+        glu-situglu_low-10L
+        mlp-waleed10
+    """
+    parts = variant.split('-')
+    if len(parts) < 2:
+        raise ValueError(f"Invalid variant format: '{variant}'. Expected: <glu|mlp>-<activation>[-<layers>L]")
+
+    prefix = parts[0]
+    if prefix not in ('glu', 'mlp'):
+        raise ValueError(f"Invalid prefix: '{prefix}'. Must be 'glu' or 'mlp'.")
+
+    # The last part might be layers like "10L"
+    last = parts[-1]
+    if last.endswith('L') and last[:-1].isdigit():
+        layers = int(last[:-1])
+        activation = '-'.join(parts[1:-1])   # everything between prefix and layers
+    else:
+        layers = None
+        activation = '-'.join(parts[1:])     # everything after prefix
+
+    if not activation:
+        raise ValueError(f"Missing activation name in variant: '{variant}'")
+
+    return prefix, activation, layers
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", required=True, help="Base YAML config")
+    parser.add_argument(
+        "--variants",
+        nargs="+",
+        required=True,
+        help="List of variants: e.g. glu-silu-10L mlp-relu-8L"
+    )
+    parser.add_argument("--push", action="store_true")
+    args = parser.parse_args()
+
+    with open(args.config) as f:
+        base = yaml.safe_load(f)
+
+    seed = base.get("training", {}).get("seed", 42)
+    set_seed(seed)
+
+    # Force all runs into a specific WandB project (read from config)
+    wandb_project = base.get("training", {}).get("wandb_project")
+    if wandb_project:
+        os.environ["WANDB_PROJECT"] = wandb_project
+        print(f"[WandB] Project locked to: {wandb_project}")
+
+    tok_name = base["model"].get("tokenizer_name", "meta-llama/Llama-2-7b-hf")
+    tokenizer = AutoTokenizer.from_pretrained(tok_name)
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    msl = base["model"].get("max_position_embeddings", 512)
+    train_ds = build_dataset(
+        tokenizer,
+        max_seq_len=msl,
+        split="train",
+        max_samples=None
+    )
+    eval_ds = build_dataset(
+        tokenizer,
+        max_seq_len=msl,
+        split="validation",
+        max_samples=None
+    )
+
+    results = []
+
+    for variant in args.variants:
+        prefix, act, layers = parse_variant(variant)
+
+        # Validation: mlp-situglu and mlp-waleed are banned
+        if prefix == "mlp" and act in ("situglu", "waleed", "situglu_low", "waleedglu_low"):
+            raise ValueError(
+                f"Activation '{act}' requires a gated architecture (GLU). "
+                f"Please use 'glu-{act}' instead."
+            )
+
+        # Build config overrides
+        cfg = copy.deepcopy(base)
+        cfg["model"]["mlp_type"] = prefix
+        cfg["model"]["activation"] = act
+        if layers is not None:
+            cfg["model"]["num_hidden_layers"] = layers
+
+        # --------------------------------------------------------------
+        # ENSURE LAYER COUNT IS ALWAYS INCLUDED IN ALL NAMES
+        # Read the final layer count from the config after any override.
+        # --------------------------------------------------------------
+        actual_layers = cfg["model"]["num_hidden_layers"]
+        variant_label = f"{prefix}-{act}-{actual_layers}L"
+
+        # Unique output directory
+        out_dir = Path(cfg["training"]["output_dir"]).parent / f"{variant_label}_run"
+        cfg["training"]["output_dir"] = str(out_dir)
+
+        # Re-seed for reproducibility across variants
+        set_seed(seed)
+
+        print(f"\n{'='*60}\n>>> Variant: {variant_label} | Out: {out_dir}\n{'='*60}")
+
+        # Instantiate model
+        config = TinyLlamaConfig(**cfg["model"])
+        model = TinyLlamaForCausalLM(config)
+        model = model.to(torch.bfloat16)
+
+        total_params = sum(p.numel() for p in model.parameters())
+        param_str = format_param_count(total_params)
+        timestamp = time.strftime("%Y%m%d-%H%M%S")
+        run_name = f"LM-{variant_label}-{param_str}-{timestamp}"
+        cfg["training"]["run_name"] = run_name
+
+        # Also update hub_model_id to include variant and layer count
+        hub_id_base = cfg["training"].get("hub_model_id", "tiny-llama-lab")
+        cfg["training"]["hub_model_id"] = f"{hub_id_base}-{variant_label}"
+
+        # Ensure a fresh WandB run – remove any global WANDB_RUN_ID
+        os.environ.pop("WANDB_RUN_ID", None)
+
+        trainer = create_trainer(model, tokenizer, cfg, train_ds, eval_ds)
+
+        try:
+            trainer.train()
+            metrics = trainer.evaluate()
+            results.append({
+                "variant": variant_label,   # now always includes layer count
+                "eval_loss": metrics.get("eval_loss"),
+                "out": str(out_dir),
+                "run_name": run_name,
+            })
+            trainer.save_model(str(out_dir))
+            if args.push or cfg["training"].get("push_to_hub", False):
+                trainer.push_to_hub()
+        finally:
+            # Explicitly finish WandB run to avoid re‑using the same run
+            wandb.finish()
+
+    # Save summary
+    summary = Path(base["training"]["output_dir"]).parent / "sweep_summary.json"
+    summary.write_text(json.dumps(results, indent=2))
+    print("\nSweep complete:")
+    for r in results:
+        print(f"  {r['variant']:20s}  eval_loss={r['eval_loss']:.4f}")
+
+
+if __name__ == "__main__":
+    main()

zain/Activation/train.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Train one TinyLlama variant from a YAML config."""
+import argparse
+import yaml
+import os
+import torch
+
+from transformers import AutoTokenizer, set_seed
+from exp import TinyLlamaConfig, TinyLlamaForCausalLM, build_dataset, create_trainer
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", required=True, help="Path to YAML config")
+    parser.add_argument("--push", action="store_true", help="Push final model to HF Hub")
+    args = parser.parse_args()
+
+    with open(args.config) as f:
+        cfg = yaml.safe_load(f)
+
+    # Explicit seed before any randomness
+    seed = cfg.get("training", {}).get("seed", 42)
+    set_seed(seed)
+
+    # -------------------------------------------------------------------------
+    # NEW: Force run into a specific WandB project (read from config).
+    # -------------------------------------------------------------------------
+    wandb_project = cfg.get("training", {}).get("wandb_project")
+    if wandb_project:
+        os.environ["WANDB_PROJECT"] = wandb_project
+        print(f"[WandB] Project locked to: {wandb_project}")
+    # -------------------------------------------------------------------------
+
+    model_cfg = cfg["model"]
+    train_cfg = cfg.get("training", {})
+
+    # Tokenizer
+    tok_name = model_cfg.pop("tokenizer_name", "meta-llama/Llama-2-7b-hf")
+    tokenizer = AutoTokenizer.from_pretrained(tok_name)
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    # Model
+    tiny_config = TinyLlamaConfig(**model_cfg)
+    model = TinyLlamaForCausalLM(tiny_config)
+    model = model.to(torch.bfloat16)
+
+    n_params = sum(p.numel() for p in model.parameters()) / 1e6
+    print(f"Model: {n_params:.2f}M params | MLP type: {tiny_config.mlp_type} | Activation: {tiny_config.activation}")
+
+    # Data
+    msl = model_cfg.get("max_position_embeddings", 512)
+    train_ds = build_dataset(
+        tokenizer,
+        max_seq_len=msl,
+        split="train",
+        max_samples=None
+    )
+    eval_ds = build_dataset(
+        tokenizer,
+        max_seq_len=msl,
+        split="validation",
+        max_samples=None
+    )
+
+    # Train
+    trainer = create_trainer(model, tokenizer, cfg, train_ds, eval_ds)
+    trainer.train()
+
+    # Save & push
+    out = train_cfg.get("output_dir", "./out")
+    trainer.save_model(out)
+    if args.push or train_cfg.get("push_to_hub", False):
+        trainer.push_to_hub()
+    print(f"Done. Artifacts in {out}")
+
+
+if __name__ == "__main__":
+    main()

zain/Activation/wandb/debug-internal.log

@@ -0,0 +1,13 @@
+{"time":"2026-08-12T18:21:26.321661629Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-12T18:21:26.321798801Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-12T18:21:26.578546502Z","level":"INFO","msg":"stream: created new stream","id":"vyycsg6r"}
+{"time":"2026-08-12T18:21:26.578636888Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-12T18:21:26.578724462Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-12T18:21:26.578734628Z","level":"INFO","msg":"writer: started","stream_id":"vyycsg6r"}
+{"time":"2026-08-12T18:21:26.578760021Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-12T18:21:27.883572309Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:21:27.993870368Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:21:42.884685513Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":2,"console_offset":0,"console_lines":2,"uploaded_len":2}
+{"time":"2026-08-12T18:21:43.002439917Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:21:57.884556428Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":2,"events_lines":2,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T18:21:58.005057203Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}

zain/Activation/wandb/debug.log

@@ -0,0 +1,20 @@
+2026-08-12 18:21:26,320 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_182126-vyycsg6r/logs/debug.log
+2026-08-12 18:21:26,320 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_182126-vyycsg6r/logs/debug-internal.log
+2026-08-12 18:21:26,320 INFO    MainThread:1371664 [wandb_init.py:init():772] calling init triggers
+2026-08-12 18:21:26,320 INFO    MainThread:1371664 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-12 18:21:26,320 INFO    MainThread:1371664 [wandb_init.py:init():820] starting backend
+2026-08-12 18:21:26,320 INFO    MainThread:1371664 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-12 18:21:26,320 INFO    MainThread:1371664 [wandb_init.py:init():835] sending inform_init request
+2026-08-12 18:21:26,579 INFO    MainThread:1371664 [wandb_init.py:init():840] backend started and connected
+2026-08-12 18:21:26,580 INFO    MainThread:1371664 [wandb_init.py:init():910] updated telemetry
+2026-08-12 18:21:26,586 INFO    MainThread:1371664 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-12 18:21:26,892 INFO    MainThread:1371664 [wandb_init.py:init():978] starting run threads in backend
+2026-08-12 18:21:26,969 INFO    MainThread:1371664 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-12 18:21:26,969 INFO    MainThread:1371664 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-12 18:21:26,969 INFO    MainThread:1371664 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-12 18:21:26,969 INFO    MainThread:1371664 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-12 18:21:26,970 INFO    MainThread:1371664 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-12 18:21:26,971 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 150, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'gelu', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-gelu-150L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.005, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-gelu-150L-25.1M-20260812-182125', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-gelu-150L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-12 18:21:26,978 INFO    MainThread:1371664 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 25138816 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c5efbb9090>>
+2026-08-12 18:21:26,979 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 25138816 None

zain/Activation/wandb/run-20260812_175304-hrggki9k/files/config.yaml

@@ -0,0 +1,445 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            nyupeef48exzv166j8gx7nmchredluzy:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline150L.yaml
+                    - --variants
+                    - glu-silu
+                    - glu-relu
+                    - glu-gelu
+                    - glu-sigmoid
+                    - glu-linear
+                    - glu-waleed10
+                    - mlp-waleed10
+                    - glu-silu-waleed10
+                    - mlp-silu-waleed10
+                    - glu-situglu
+                    - glu-waleed
+                    - glu-situglu_low
+                    - glu-waleedglu_low
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "716176670720"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: c53713bcba08d1849136dad91288c4b82040634e
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-12T17:53:04.074150Z"
+                writerId: nyupeef48exzv166j8gx7nmchredluzy
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: silu
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 50
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-glu-silu-150L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.005
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 0
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 25138816
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 150
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-silu-150L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 128
+per_device_train_batch_size:
+    value: 64
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-silu-150L-25.1M-20260812-175302
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 1000
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 200
+weight_decay:
+    value: 0

zain/Activation/wandb/run-20260812_175304-hrggki9k/files/output.log

@@ -0,0 +1,85 @@
+[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!
+[INFO] Causal mask (float with -inf) applied to all attention layers.
+                                                    
+{'loss': '7.966', 'grad_norm': '1.258', 'learning_rate': '0.000475', 'epoch': '0.001348', 'train/total_time_seconds': '7.768', 'train/time_per_step_avg': '0.3884', 'train/epoch_time_elapsed': '8.87', 'train/estimated_remaining_minutes': '6.344'}
+{'loss': '6.834', 'grad_norm': '0.875', 'learning_rate': '0.000975', 'epoch': '0.002696', 'train/total_time_seconds': '14.69', 'train/time_per_step_avg': '0.3673', 'train/epoch_time_elapsed': '16.77', 'train/estimated_remaining_minutes': '5.877'}
+                                               
+{'eval_loss': '6.037', 'eval_runtime': '20.95', 'eval_samples_per_second': '454.8', 'eval_steps_per_second': '3.581', 'epoch': '0.00337', 'train/total_time_seconds': '18.17', 'train/time_per_step_avg': '0.3634', 'train/epoch_time_elapsed': '41.69', 'train/estimated_remaining_minutes': '5.753'}
+{'loss': '6.079', 'grad_norm': '0.2305', 'learning_rate': '0.001475', 'epoch': '0.004044', 'train/total_time_seconds': '21.63', 'train/time_per_step_avg': '0.3605', 'train/epoch_time_elapsed': '45.63', 'train/estimated_remaining_minutes': '5.648'}
+{'loss': '6.011', 'grad_norm': '0.1201', 'learning_rate': '0.001975', 'epoch': '0.005393', 'train/total_time_seconds': '28.59', 'train/time_per_step_avg': '0.3574', 'train/epoch_time_elapsed': '53.58', 'train/estimated_remaining_minutes': '5.48'}
+{'loss': '5.894', 'grad_norm': '0.5312', 'learning_rate': '0.002475', 'epoch': '0.006741', 'train/total_time_seconds': '35.54', 'train/time_per_step_avg': '0.3554', 'train/epoch_time_elapsed': '61.52', 'train/estimated_remaining_minutes': '5.331'}
+{'eval_loss': '5.803', 'eval_runtime': '21', 'eval_samples_per_second': '453.6', 'eval_steps_per_second': '3.571', 'epoch': '0.006741', 'train/total_time_seconds': '35.54', 'train/time_per_step_avg': '0.3554', 'train/epoch_time_elapsed': '82.52', 'train/estimated_remaining_minutes': '5.331'}
+{'loss': '5.78', 'grad_norm': '0.3184', 'learning_rate': '0.002975', 'epoch': '0.008089', 'train/total_time_seconds': '42.51', 'train/time_per_step_avg': '0.3474', 'train/epoch_time_elapsed': '90.48', 'train/estimated_remaining_minutes': '5.195'}
+{'loss': '5.595', 'grad_norm': '0.2012', 'learning_rate': '0.003475', 'epoch': '0.009437', 'train/total_time_seconds': '49.74', 'train/time_per_step_avg': '0.3505', 'train/epoch_time_elapsed': '98.69', 'train/estimated_remaining_minutes': '5.092'}
+{'eval_loss': '5.564', 'eval_runtime': '21.06', 'eval_samples_per_second': '452.3', 'eval_steps_per_second': '3.561', 'epoch': '0.01011', 'train/total_time_seconds': '53.27', 'train/time_per_step_avg': '0.351', 'train/epoch_time_elapsed': '123.8', 'train/estimated_remaining_minutes': '5.031'}
+{'loss': '5.554', 'grad_norm': '0.2383', 'learning_rate': '0.003975', 'epoch': '0.01079', 'train/total_time_seconds': '56.73', 'train/time_per_step_avg': '0.351', 'train/epoch_time_elapsed': '127.7', 'train/estimated_remaining_minutes': '4.964'}
+{'loss': '5.472', 'grad_norm': '0.3047', 'learning_rate': '0.004475', 'epoch': '0.01213', 'train/total_time_seconds': '63.65', 'train/time_per_step_avg': '0.3506', 'train/epoch_time_elapsed': '135.6', 'train/estimated_remaining_minutes': '4.833'}
+{'loss': '5.362', 'grad_norm': '0.2109', 'learning_rate': '0.004975', 'epoch': '0.01348', 'train/total_time_seconds': '70.62', 'train/time_per_step_avg': '0.3508', 'train/epoch_time_elapsed': '143.5', 'train/estimated_remaining_minutes': '4.708'}
+{'eval_loss': '5.278', 'eval_runtime': '21.03', 'eval_samples_per_second': '453', 'eval_steps_per_second': '3.566', 'epoch': '0.01348', 'train/total_time_seconds': '70.62', 'train/time_per_step_avg': '0.3508', 'train/epoch_time_elapsed': '164.6', 'train/estimated_remaining_minutes': '4.708'}
+{'loss': '5.181', 'grad_norm': '0.3945', 'learning_rate': '0.005', 'epoch': '0.01483', 'train/total_time_seconds': '77.71', 'train/time_per_step_avg': '0.352', 'train/epoch_time_elapsed': '172.6', 'train/estimated_remaining_minutes': '4.592'}
+{'loss': '5.014', 'grad_norm': '0.2754', 'learning_rate': '0.005', 'epoch': '0.01618', 'train/total_time_seconds': '84.77', 'train/time_per_step_avg': '0.3503', 'train/epoch_time_elapsed': '180.7', 'train/estimated_remaining_minutes': '4.474'}
+{'eval_loss': '4.766', 'eval_runtime': '21', 'eval_samples_per_second': '453.8', 'eval_steps_per_second': '3.572', 'epoch': '0.01685', 'train/total_time_seconds': '88.31', 'train/time_per_step_avg': '0.3504', 'train/epoch_time_elapsed': '205.7', 'train/estimated_remaining_minutes': '4.415'}
+{'loss': '4.78', 'grad_norm': '0.7578', 'learning_rate': '0.005', 'epoch': '0.01753', 'train/total_time_seconds': '92.08', 'train/time_per_step_avg': '0.3535', 'train/epoch_time_elapsed': '210', 'train/estimated_remaining_minutes': '4.368'}
+{'loss': '4.661', 'grad_norm': '0.3086', 'learning_rate': '0.005', 'epoch': '0.01887', 'train/total_time_seconds': '99.16', 'train/time_per_step_avg': '0.355', 'train/epoch_time_elapsed': '218.1', 'train/estimated_remaining_minutes': '4.25'}
+{'loss': '4.492', 'grad_norm': '0.6562', 'learning_rate': '0.005', 'epoch': '0.02022', 'train/total_time_seconds': '106.2', 'train/time_per_step_avg': '0.3563', 'train/epoch_time_elapsed': '226.1', 'train/estimated_remaining_minutes': '4.132'}
+{'eval_loss': '4.413', 'eval_runtime': '21.14', 'eval_samples_per_second': '450.7', 'eval_steps_per_second': '3.548', 'epoch': '0.02022', 'train/total_time_seconds': '106.2', 'train/time_per_step_avg': '0.3563', 'train/epoch_time_elapsed': '247.3', 'train/estimated_remaining_minutes': '4.132'}
+{'loss': '4.335', 'grad_norm': '0.3301', 'learning_rate': '0.005', 'epoch': '0.02157', 'train/total_time_seconds': '113.3', 'train/time_per_step_avg': '0.3559', 'train/epoch_time_elapsed': '255.3', 'train/estimated_remaining_minutes': '4.012'}
+{'loss': '4.257', 'grad_norm': '0.8984', 'learning_rate': '0.005', 'epoch': '0.02292', 'train/total_time_seconds': '120.3', 'train/time_per_step_avg': '0.3557', 'train/epoch_time_elapsed': '263.4', 'train/estimated_remaining_minutes': '3.893'}
+{'eval_loss': '4.176', 'eval_runtime': '21.17', 'eval_samples_per_second': '450', 'eval_steps_per_second': '3.543', 'epoch': '0.02359', 'train/total_time_seconds': '123.9', 'train/time_per_step_avg': '0.3556', 'train/epoch_time_elapsed': '288.5', 'train/estimated_remaining_minutes': '3.834'}
+{'loss': '4.192', 'grad_norm': '0.2891', 'learning_rate': '0.005', 'epoch': '0.02427', 'train/total_time_seconds': '127.4', 'train/time_per_step_avg': '0.3534', 'train/epoch_time_elapsed': '292.6', 'train/estimated_remaining_minutes': '3.775'}
+{'loss': '4.055', 'grad_norm': '0.7188', 'learning_rate': '0.005', 'epoch': '0.02562', 'train/total_time_seconds': '134.5', 'train/time_per_step_avg': '0.3532', 'train/epoch_time_elapsed': '300.6', 'train/estimated_remaining_minutes': '3.657'}
+{'loss': '4.114', 'grad_norm': '0.6367', 'learning_rate': '0.005', 'epoch': '0.02696', 'train/total_time_seconds': '141.8', 'train/time_per_step_avg': '0.3551', 'train/epoch_time_elapsed': '308.9', 'train/estimated_remaining_minutes': '3.544'}
+{'eval_loss': '4.115', 'eval_runtime': '21.01', 'eval_samples_per_second': '453.5', 'eval_steps_per_second': '3.57', 'epoch': '0.02696', 'train/total_time_seconds': '141.8', 'train/time_per_step_avg': '0.3551', 'train/epoch_time_elapsed': '329.9', 'train/estimated_remaining_minutes': '3.544'}
+{'loss': '4.029', 'grad_norm': '0.3223', 'learning_rate': '0.005', 'epoch': '0.02831', 'train/total_time_seconds': '148.7', 'train/time_per_step_avg': '0.3544', 'train/epoch_time_elapsed': '337.8', 'train/estimated_remaining_minutes': '3.423'}
+{'loss': '3.93', 'grad_norm': '0.2412', 'learning_rate': '0.005', 'epoch': '0.02966', 'train/total_time_seconds': '155.7', 'train/time_per_step_avg': '0.3533', 'train/epoch_time_elapsed': '345.7', 'train/estimated_remaining_minutes': '3.302'}
+{'eval_loss': '3.827', 'eval_runtime': '20.98', 'eval_samples_per_second': '454.2', 'eval_steps_per_second': '3.575', 'epoch': '0.03033', 'train/total_time_seconds': '159.1', 'train/time_per_step_avg': '0.3527', 'train/epoch_time_elapsed': '370.7', 'train/estimated_remaining_minutes': '3.242'}
+{'loss': '3.827', 'grad_norm': '0.248', 'learning_rate': '0.005', 'epoch': '0.03101', 'train/total_time_seconds': '162.7', 'train/time_per_step_avg': '0.3524', 'train/epoch_time_elapsed': '374.7', 'train/estimated_remaining_minutes': '3.182'}
+{'loss': '3.767', 'grad_norm': '0.252', 'learning_rate': '0.005', 'epoch': '0.03236', 'train/total_time_seconds': '169.8', 'train/time_per_step_avg': '0.353', 'train/epoch_time_elapsed': '382.8', 'train/estimated_remaining_minutes': '3.065'}
+{'loss': '3.703', 'grad_norm': '0.418', 'learning_rate': '0.005', 'epoch': '0.0337', 'train/total_time_seconds': '176.8', 'train/time_per_step_avg': '0.351', 'train/epoch_time_elapsed': '390.9', 'train/estimated_remaining_minutes': '2.947'}
+{'eval_loss': '3.682', 'eval_runtime': '21.07', 'eval_samples_per_second': '452.3', 'eval_steps_per_second': '3.56', 'epoch': '0.0337', 'train/total_time_seconds': '176.8', 'train/time_per_step_avg': '0.351', 'train/epoch_time_elapsed': '411.9', 'train/estimated_remaining_minutes': '2.947'}
+{'loss': '3.661', 'grad_norm': '0.3008', 'learning_rate': '0.005', 'epoch': '0.03505', 'train/total_time_seconds': '183.8', 'train/time_per_step_avg': '0.3506', 'train/epoch_time_elapsed': '419.9', 'train/estimated_remaining_minutes': '2.828'}
+{'loss': '3.6', 'grad_norm': '0.3203', 'learning_rate': '0.005', 'epoch': '0.0364', 'train/total_time_seconds': '191', 'train/time_per_step_avg': '0.3538', 'train/epoch_time_elapsed': '428.1', 'train/estimated_remaining_minutes': '2.712'}
+{'eval_loss': '3.577', 'eval_runtime': '21.01', 'eval_samples_per_second': '453.4', 'eval_steps_per_second': '3.569', 'epoch': '0.03707', 'train/total_time_seconds': '194.5', 'train/time_per_step_avg': '0.3538', 'train/epoch_time_elapsed': '453.1', 'train/estimated_remaining_minutes': '2.653'}
+{'loss': '3.556', 'grad_norm': '0.3867', 'learning_rate': '0.005', 'epoch': '0.03775', 'train/total_time_seconds': '198', 'train/time_per_step_avg': '0.3533', 'train/epoch_time_elapsed': '457', 'train/estimated_remaining_minutes': '2.593'}
+{'loss': '3.66', 'grad_norm': '0.7148', 'learning_rate': '0.005', 'epoch': '0.0391', 'train/total_time_seconds': '205', 'train/time_per_step_avg': '0.3521', 'train/epoch_time_elapsed': '465', 'train/estimated_remaining_minutes': '2.474'}
+{'loss': '3.643', 'grad_norm': '0.2773', 'learning_rate': '0.005', 'epoch': '0.04044', 'train/total_time_seconds': '212', 'train/time_per_step_avg': '0.3512', 'train/epoch_time_elapsed': '473', 'train/estimated_remaining_minutes': '2.355'}
+{'eval_loss': '3.569', 'eval_runtime': '21.07', 'eval_samples_per_second': '452.2', 'eval_steps_per_second': '3.56', 'epoch': '0.04044', 'train/total_time_seconds': '212', 'train/time_per_step_avg': '0.3512', 'train/epoch_time_elapsed': '494.1', 'train/estimated_remaining_minutes': '2.355'}
+{'loss': '3.522', 'grad_norm': '0.2812', 'learning_rate': '0.005', 'epoch': '0.04179', 'train/total_time_seconds': '219.1', 'train/time_per_step_avg': '0.3534', 'train/epoch_time_elapsed': '502.2', 'train/estimated_remaining_minutes': '2.238'}
+{'loss': '3.495', 'grad_norm': '0.5', 'learning_rate': '0.005', 'epoch': '0.04314', 'train/total_time_seconds': '226.2', 'train/time_per_step_avg': '0.3519', 'train/epoch_time_elapsed': '510.3', 'train/estimated_remaining_minutes': '2.121'}
+{'eval_loss': '3.45', 'eval_runtime': '21.05', 'eval_samples_per_second': '452.6', 'eval_steps_per_second': '3.563', 'epoch': '0.04382', 'train/total_time_seconds': '229.8', 'train/time_per_step_avg': '0.3526', 'train/epoch_time_elapsed': '535.4', 'train/estimated_remaining_minutes': '2.062'}
+{'loss': '3.442', 'grad_norm': '0.4238', 'learning_rate': '0.005', 'epoch': '0.04449', 'train/total_time_seconds': '233.3', 'train/time_per_step_avg': '0.3533', 'train/epoch_time_elapsed': '539.5', 'train/estimated_remaining_minutes': '2.003'}
+{'loss': '3.431', 'grad_norm': '0.2812', 'learning_rate': '0.005', 'epoch': '0.04584', 'train/total_time_seconds': '240.6', 'train/time_per_step_avg': '0.3565', 'train/epoch_time_elapsed': '547.7', 'train/estimated_remaining_minutes': '1.887'}
+{'loss': '3.38', 'grad_norm': '0.2969', 'learning_rate': '0.005', 'epoch': '0.04719', 'train/total_time_seconds': '247.8', 'train/time_per_step_avg': '0.3583', 'train/epoch_time_elapsed': '555.9', 'train/estimated_remaining_minutes': '1.77'}
+{'eval_loss': '3.382', 'eval_runtime': '21.15', 'eval_samples_per_second': '450.5', 'eval_steps_per_second': '3.547', 'epoch': '0.04719', 'train/total_time_seconds': '247.8', 'train/time_per_step_avg': '0.3583', 'train/epoch_time_elapsed': '577', 'train/estimated_remaining_minutes': '1.77'}
+{'loss': '3.357', 'grad_norm': '0.4512', 'learning_rate': '0.005', 'epoch': '0.04853', 'train/total_time_seconds': '254.9', 'train/time_per_step_avg': '0.3577', 'train/epoch_time_elapsed': '585.1', 'train/estimated_remaining_minutes': '1.652'}
+{'loss': '3.327', 'grad_norm': '0.3145', 'learning_rate': '0.005', 'epoch': '0.04988', 'train/total_time_seconds': '262.1', 'train/time_per_step_avg': '0.3588', 'train/epoch_time_elapsed': '593.4', 'train/estimated_remaining_minutes': '1.535'}
+{'eval_loss': '3.327', 'eval_runtime': '21.1', 'eval_samples_per_second': '451.5', 'eval_steps_per_second': '3.555', 'epoch': '0.05056', 'train/total_time_seconds': '265.7', 'train/time_per_step_avg': '0.3587', 'train/epoch_time_elapsed': '618.5', 'train/estimated_remaining_minutes': '1.476'}
+{'loss': '3.323', 'grad_norm': '0.3379', 'learning_rate': '0.005', 'epoch': '0.05123', 'train/total_time_seconds': '269.4', 'train/time_per_step_avg': '0.3605', 'train/epoch_time_elapsed': '622.8', 'train/estimated_remaining_minutes': '1.418'}
+{'loss': '3.313', 'grad_norm': '0.3281', 'learning_rate': '0.005', 'epoch': '0.05258', 'train/total_time_seconds': '276.4', 'train/time_per_step_avg': '0.358', 'train/epoch_time_elapsed': '630.8', 'train/estimated_remaining_minutes': '1.299'}
+{'loss': '3.281', 'grad_norm': '0.3164', 'learning_rate': '0.005', 'epoch': '0.05393', 'train/total_time_seconds': '283.5', 'train/time_per_step_avg': '0.3572', 'train/epoch_time_elapsed': '638.9', 'train/estimated_remaining_minutes': '1.181'}
+{'eval_loss': '3.28', 'eval_runtime': '21.12', 'eval_samples_per_second': '451.1', 'eval_steps_per_second': '3.551', 'epoch': '0.05393', 'train/total_time_seconds': '283.5', 'train/time_per_step_avg': '0.3572', 'train/epoch_time_elapsed': '660', 'train/estimated_remaining_minutes': '1.181'}
+{'loss': '3.288', 'grad_norm': '0.7617', 'learning_rate': '0.005', 'epoch': '0.05527', 'train/total_time_seconds': '290.8', 'train/time_per_step_avg': '0.3588', 'train/epoch_time_elapsed': '668.2', 'train/estimated_remaining_minutes': '1.064'}
+{'loss': '3.274', 'grad_norm': '0.2891', 'learning_rate': '0.005', 'epoch': '0.05662', 'train/total_time_seconds': '298', 'train/time_per_step_avg': '0.3586', 'train/epoch_time_elapsed': '676.4', 'train/estimated_remaining_minutes': '0.946'}
+{'eval_loss': '3.243', 'eval_runtime': '21.02', 'eval_samples_per_second': '453.1', 'eval_steps_per_second': '3.567', 'epoch': '0.0573', 'train/total_time_seconds': '301.5', 'train/time_per_step_avg': '0.3588', 'train/epoch_time_elapsed': '701.5', 'train/estimated_remaining_minutes': '0.8869'}
+{'loss': '3.226', 'grad_norm': '0.3008', 'learning_rate': '0.005', 'epoch': '0.05797', 'train/total_time_seconds': '305.1', 'train/time_per_step_avg': '0.3573', 'train/epoch_time_elapsed': '705.5', 'train/estimated_remaining_minutes': '0.8278'}
+{'loss': '3.212', 'grad_norm': '0.7734', 'learning_rate': '0.005', 'epoch': '0.05932', 'train/total_time_seconds': '312.2', 'train/time_per_step_avg': '0.3581', 'train/epoch_time_elapsed': '713.7', 'train/estimated_remaining_minutes': '0.7096'}
+{'loss': '3.215', 'grad_norm': '0.4453', 'learning_rate': '0.005', 'epoch': '0.06067', 'train/total_time_seconds': '319.3', 'train/time_per_step_avg': '0.3582', 'train/epoch_time_elapsed': '721.7', 'train/estimated_remaining_minutes': '0.5914'}
+{'eval_loss': '3.204', 'eval_runtime': '21.09', 'eval_samples_per_second': '451.8', 'eval_steps_per_second': '3.557', 'epoch': '0.06067', 'train/total_time_seconds': '319.3', 'train/time_per_step_avg': '0.3582', 'train/epoch_time_elapsed': '742.8', 'train/estimated_remaining_minutes': '0.5914'}
+{'loss': '3.2', 'grad_norm': '0.4297', 'learning_rate': '0.005', 'epoch': '0.06202', 'train/total_time_seconds': '326.4', 'train/time_per_step_avg': '0.3563', 'train/epoch_time_elapsed': '750.9', 'train/estimated_remaining_minutes': '0.4731'}
+{'loss': '3.186', 'grad_norm': '0.2695', 'learning_rate': '0.005', 'epoch': '0.06336', 'train/total_time_seconds': '333.7', 'train/time_per_step_avg': '0.3568', 'train/epoch_time_elapsed': '759.1', 'train/estimated_remaining_minutes': '0.355'}
+{'eval_loss': '3.177', 'eval_runtime': '21.38', 'eval_samples_per_second': '445.6', 'eval_steps_per_second': '3.508', 'epoch': '0.06404', 'train/total_time_seconds': '337.2', 'train/time_per_step_avg': '0.3566', 'train/epoch_time_elapsed': '784.5', 'train/estimated_remaining_minutes': '0.2958'}
+{'loss': '3.168', 'grad_norm': '0.4004', 'learning_rate': '0.005', 'epoch': '0.06471', 'train/total_time_seconds': '340.7', 'train/time_per_step_avg': '0.3559', 'train/epoch_time_elapsed': '788.5', 'train/estimated_remaining_minutes': '0.2366'}
+{'loss': '3.164', 'grad_norm': '0.4941', 'learning_rate': '0.005', 'epoch': '0.06606', 'train/total_time_seconds': '347.7', 'train/time_per_step_avg': '0.3542', 'train/epoch_time_elapsed': '796.4', 'train/estimated_remaining_minutes': '0.1183'}
+{'loss': '3.157', 'grad_norm': '0.6055', 'learning_rate': '0.005', 'epoch': '0.06741', 'train/total_time_seconds': '354.7', 'train/time_per_step_avg': '0.3532', 'train/epoch_time_elapsed': '804.4', 'train/estimated_remaining_minutes': '0'}
+{'eval_loss': '3.169', 'eval_runtime': '20.96', 'eval_samples_per_second': '454.6', 'eval_steps_per_second': '3.579', 'epoch': '0.06741', 'train/total_time_seconds': '354.7', 'train/time_per_step_avg': '0.3532', 'train/epoch_time_elapsed': '825.4', 'train/estimated_remaining_minutes': '0'}
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00,  9.22it/s]
+100%|██████████| 1000/1000 [13:46<00:00,  1.21it/s]0:00,  9.30it/s]
+{'train_runtime': '826.5', 'train_samples_per_second': '77.43', 'train_steps_per_second': '1.21', 'train_loss': '4.179', 'epoch': '0.06741', 'train/total_time_seconds': '354.7', 'train/time_per_step_avg': '0.3532', 'train/epoch_time_elapsed': '825.8', 'train/estimated_remaining_minutes': '0'}
+100%|██████████| 75/75 [00:20<00:00,  3.63it/s]
+[transformers] TinyLlamaForCausalLM has generative capabilities, as `prepare_inputs_for_generation` is explicitly defined. However, it doesn't directly inherit from `GenerationMixin`. From 👉v4.50👈 onwards, `PreTrainedModel` will NOT inherit from `GenerationMixin`, and this model will lose the ability to call `generate` and other related functions.
+  - If you're using `trust_remote_code=True`, you can get rid of this warning by loading the model with an auto class. See https://huggingface.co/docs/transformers/en/model_doc/auto#auto-classes
+  - If you are the owner of the model architecture code, please modify your model class such that it inherits from `GenerationMixin` (after `PreTrainedModel`, otherwise you'll get an exception).
+  - If you are not the owner of the model architecture class, please contact the model code owner to update it.
+Writing model shards: 100%|██████████| 1/1 [00:00<00:00, 11.28it/s]

zain/Activation/wandb/run-20260812_175304-hrggki9k/files/requirements.txt

@@ -0,0 +1,149 @@
+asttokens==3.0.1
+comm==0.2.3
+debugpy==1.8.21
+decorator==5.3.1
+executing==2.2.1
+nest-asyncio==1.6.0
+parso==0.8.7
+platformdirs==4.11.0
+psutil==7.2.2
+ptyprocess==0.7.0
+pure_eval==0.2.3
+Pygments==2.20.0
+pyzmq==27.1.0
+setuptools==83.0.0
+six==1.17.0
+tornado==6.5.7
+traitlets==5.15.0
+fsspec==2026.4.0
+wcwidth==0.8.2
+ipython_pygments_lexers==1.1.1
+jedi==0.20.0
+jupyter_core==5.9.1
+matplotlib-inline==0.2.2
+pexpect==4.9.0
+prompt_toolkit==3.0.53
+python-dateutil==2.9.0.post0
+stack_data==0.6.3
+wheel==0.47.0
+jupyter_client==8.9.1
+pip==26.1.2
+ipython==9.15.0
+ipykernel==7.2.0
+threadpoolctl==3.6.0
+pyparsing==3.3.2
+typing_extensions==4.15.0
+Jinja2==3.1.6
+narwhals==2.24.0
+kiwisolver==1.5.0
+joblib==1.5.3
+fonttools==4.63.0
+cycler==0.12.1
+scipy==1.17.1
+pandas==3.0.5
+contourpy==1.3.3
+scikit-learn==1.9.0
+matplotlib==3.11.1
+urllib3==2.7.0
+tqdm==4.70.0
+idna==3.18
+charset-normalizer==3.4.9
+certifi==2026.7.22
+requests==2.34.2
+seaborn==0.13.2
+uv==0.12.0
+shellingham==1.5.4
+mpmath==1.3.0
+attrs==26.1.0
+hf-xet==1.5.2
+nvidia-nccl-cu12==2.21.5
+MarkupSafe==3.0.3
+regex==2026.7.19
+importlib_metadata==9.0.0
+httpcore==1.0.9
+annotated-doc==0.0.5
+multidict==6.7.1
+aiohttp==3.14.3
+aiosignal==1.4.0
+xxhash==3.8.1
+aiohappyeyeballs==2.7.1
+mdurl==0.1.2
+cuda-toolkit==13.0.3.0
+networkx==3.6.1
+PyYAML==6.0.3
+nvidia-cufile==1.15.1.6
+typer==0.27.0
+torchaudio==2.6.0+cu124
+rich==15.0.0
+nvidia-cufft-cu12==11.2.1.3
+h11==0.16.0
+dill==0.4.1
+cuda-pathfinder==1.6.0
+filelock==3.29.0
+nvidia-nvtx-cu12==12.4.127
+httpx==0.28.1
+anyio==4.14.2
+numpy==2.4.4
+yarl==1.24.5
+click==8.4.2
+triton==3.2.0
+frozenlist==1.8.0
+zipp==4.1.0
+propcache==0.5.2
+tokenizers==0.22.2
+markdown-it-py==4.2.0
+nvidia-cuda-runtime==13.0.96
+cuda-bindings==13.3.1
+nvidia-cuda-cupti==13.0.85
+torch==2.6.0+cu124
+multiprocess==0.70.19
+pillow==12.2.0
+transformers==5.16.0.dev0
+wandb==0.28.1
+nvidia-curand==10.4.0.35
+sympy==1.13.1
+nvidia-cusparse==12.6.3.3
+nvidia-cuda-nvrtc==13.0.88
+typing-inspection==0.4.2
+nvidia-cusolver==12.0.4.66
+nvidia-cufft==12.0.0.61
+nvidia-cudnn-cu13==9.20.0.48
+nvidia-cublas==13.1.1.3
+pyarrow==25.0.0
+evaluate==0.4.6
+diffusers==0.39.0
+pydantic==2.13.4
+annotated-types==0.8.0
+protobuf==7.35.1
+sentry-sdk==2.66.1
+einops==0.8.2
+packaging==26.2
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-curand-cu12==10.3.5.147
+nvidia-cusparselt-cu12==0.6.2
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-runtime-cu12==12.4.127
+torchvision==0.21.0+cu124
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cusolver-cu12==11.6.1.9
+nvidia-cublas-cu12==12.4.5.8
+nvidia-cudnn-cu12==9.1.0.70
+huggingface_hub==1.26.0
+datasets==5.0.1
+safetensors==0.8.0
+accelerate==1.14.0
+pydantic_core==2.46.4
+ninja==1.13.0
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.7.1
+jaraco.text==4.0.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+more-itertools==10.8.0
+packaging==26.0
+platformdirs==4.4.0
+tomli==2.4.0
+wheel==0.46.3
+zipp==3.23.0

zain/Activation/wandb/run-20260812_175304-hrggki9k/files/wandb-metadata.json

@@ -0,0 +1,107 @@
+{
+  "os":  "Linux-5.15.0-126-generic-x86_64-with-glibc2.35",
+  "python":  "CPython 3.11.15",
+  "startedAt":  "2026-08-12T17:53:04.074150Z",
+  "args":  [
+    "--config",
+    "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline150L.yaml",
+    "--variants",
+    "glu-silu",
+    "glu-relu",
+    "glu-gelu",
+    "glu-sigmoid",
+    "glu-linear",
+    "glu-waleed10",
+    "mlp-waleed10",
+    "glu-silu-waleed10",
+    "mlp-silu-waleed10",
+    "glu-situglu",
+    "glu-waleed",
+    "glu-situglu_low",
+    "glu-waleedglu_low"
+  ],
+  "program":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py",
+  "codePath":  "sweep.py",
+  "codePathLocal":  "sweep.py",
+  "git":  {
+    "remote":  "https://github.com/w-ahmad1a10/Activation.git",
+    "commit":  "c53713bcba08d1849136dad91288c4b82040634e"
+  },
+  "email":  "deepnevro@gmail.com",
+  "root":  "/mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation",
+  "host":  "deeplens-k3s-node1",
+  "executable":  "/mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python",
+  "cpu_count":  112,
+  "cpu_count_logical":  224,
+  "gpu":  "NVIDIA H100 80GB HBM3",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "1560765693952",
+      "used":  "716176670720"
+    }
+  },
+  "memory":  {
+    "total":  "2164089937920"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-39c684a5-fde6-83d7-1663-0859795881ae"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-132944c4-b689-2b5f-89a4-d730401677ab"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-bc6c3e3c-9b90-09ca-c034-774961847c54"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9"
+    },
+    {
+      "name":  "NVIDIA H100 80GB HBM3",
+      "memoryTotal":  "85520809984",
+      "cudaCores":  16896,
+      "architecture":  "Hopper",
+      "uuid":  "GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea"
+    }
+  ],
+  "cudaVersion":  "12.4",
+  "writerId":  "nyupeef48exzv166j8gx7nmchredluzy"
+}

zain/Activation/wandb/run-20260812_175304-hrggki9k/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":847},"train/loss":3.1570945739746095,"train/grad_norm":0.60546875,"train_runtime":826.5493,"eval/loss":3.1693739891052246,"train/train/total_time_seconds":354.6520741917193,"_timestamp":1.7865580315405643e+09,"total_flos":4.839413121024e+15,"train/learning_rate":0.005,"train/train/epoch_time_elapsed":846.7927529215813,"_runtime":847,"train/train/time_per_step_avg":0.3532183859869838,"train_loss":4.179319068908692,"train/train/estimated_remaining_minutes":0,"_step":71,"train_samples_per_second":77.43,"eval/steps_per_second":3.569,"train/global_step":1000,"train/epoch":0.06740815638692282,"eval/samples_per_second":453.385,"eval/runtime":21.013,"train_steps_per_second":1.21}

zain/Activation/wandb/run-20260812_175304-hrggki9k/logs/debug-core.log

@@ -0,0 +1,34 @@
+{"time":"2026-08-12T17:51:29.496049185Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp7lru5pey/port-1355517.txt","pid":1355517,"detached":false,"idle-timeout":600000000000,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-08-12T17:51:29.496747364Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":1355517}
+{"time":"2026-08-12T17:51:29.49671311Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-1355517-1357782-3052194627/socket","Net":"unix"}}
+{"time":"2026-08-12T17:51:29.674347603Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-08-12T17:52:11.162138091Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"2(@)"}
+{"time":"2026-08-12T17:52:11.242645021Z","level":"INFO","msg":"handleInformInit: received","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:11.508233203Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:17.001520061Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"6684d59dll8w"}
+{"time":"2026-08-12T17:52:32.412124918Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"6684d59dll8w"}
+{"time":"2026-08-12T17:52:32.887487538Z","level":"INFO","msg":"connection: cancelling request","id":"2(@)","requestId":"6684d59dll8w"}
+{"time":"2026-08-12T17:52:32.888505Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:32.889313366Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"ycyhhejh","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597228944Z","level":"INFO","msg":"connection: closing","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597314364Z","level":"INFO","msg":"connection: closed successfully","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597242804Z","level":"INFO","msg":"processOutgoingData: finished","id":"2(@)"}
+{"time":"2026-08-12T17:52:34.597322441Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"2(@)"}
+{"time":"2026-08-12T17:53:03.98517338Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"3(@)"}
+{"time":"2026-08-12T17:53:04.076747848Z","level":"INFO","msg":"handleInformInit: received","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T17:53:04.339071164Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T17:53:09.739880255Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tj6i5mzy43l0"}
+{"time":"2026-08-12T18:07:11.667315857Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tj6i5mzy43l0"}
+{"time":"2026-08-12T18:07:12.178747012Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"tj6i5mzy43l0"}
+{"time":"2026-08-12T18:07:12.180598654Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T18:07:12.181733156Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"hrggki9k","id":"3(@)"}
+{"time":"2026-08-12T18:07:13.412475061Z","level":"INFO","msg":"handleInformInit: received","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:07:13.679314069Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:07:19.044675117Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"atkqmrzkldx8"}
+{"time":"2026-08-12T18:21:24.445193258Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"atkqmrzkldx8"}
+{"time":"2026-08-12T18:21:25.196343989Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"atkqmrzkldx8"}
+{"time":"2026-08-12T18:21:25.198067009Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:21:25.199000312Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"utqzlg67","id":"3(@)"}
+{"time":"2026-08-12T18:21:26.321522195Z","level":"INFO","msg":"handleInformInit: received","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:21:26.578729863Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"vyycsg6r","id":"3(@)"}
+{"time":"2026-08-12T18:21:31.999599169Z","level":"INFO","msg":"connection: cancelling request","id":"3(@)","requestId":"h28g0ntapnl9"}

zain/Activation/wandb/run-20260812_175304-hrggki9k/logs/debug-internal.log

@@ -0,0 +1,129 @@
+{"time":"2026-08-12T17:53:04.076899017Z","level":"INFO","msg":"wandb-core"}
+{"time":"2026-08-12T17:53:04.077036839Z","level":"INFO","msg":"stream: starting","core version":"0.28.1"}
+{"time":"2026-08-12T17:53:04.338896465Z","level":"INFO","msg":"stream: created new stream","id":"hrggki9k"}
+{"time":"2026-08-12T17:53:04.338980088Z","level":"INFO","msg":"handler: started"}
+{"time":"2026-08-12T17:53:04.339064128Z","level":"INFO","msg":"stream: started"}
+{"time":"2026-08-12T17:53:04.339075647Z","level":"INFO","msg":"writer: started","stream_id":"hrggki9k"}
+{"time":"2026-08-12T17:53:04.339102826Z","level":"INFO","msg":"sender: started"}
+{"time":"2026-08-12T17:53:04.923957807Z","level":"INFO","msg":"filestream: sending request","total_files":1,"console_offset":0,"console_lines":1}
+{"time":"2026-08-12T17:53:05.038683027Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:53:19.924410071Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":0,"history_lines":1,"events_offset":0,"events_lines":2,"console_offset":1,"console_lines":3,"uploaded_len":2}
+{"time":"2026-08-12T17:53:20.031904396Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:53:34.924344715Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":1,"history_lines":1,"events_offset":2,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:53:35.034525535Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:53:49.924189652Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":2,"history_lines":1,"events_offset":4,"events_lines":1,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:53:50.033021307Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:54:04.92424093Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":3,"history_lines":2,"events_offset":5,"events_lines":3,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:54:05.042251156Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:54:19.924260321Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":5,"history_lines":1,"events_offset":8,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:54:20.043281164Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:54:34.924379827Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":6,"history_lines":1,"events_offset":10,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:54:35.043997534Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:54:49.924601407Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":7,"history_lines":2,"events_offset":12,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:54:50.038107004Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:55:04.924131125Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":14,"events_lines":2,"console_offset":4,"console_lines":9}
+{"time":"2026-08-12T17:55:05.060356783Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:55:19.92445297Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":9,"history_lines":2,"events_offset":16,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:55:20.055452607Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:55:34.924556457Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":11,"history_lines":2,"events_offset":18,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:55:35.030920175Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:55:49.924611904Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":13,"history_lines":1,"events_offset":20,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:55:50.026253556Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:56:04.924982252Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":14,"history_lines":1,"events_offset":22,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:56:05.053212874Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:56:19.924105547Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":15,"history_lines":1,"events_offset":24,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:56:20.067438584Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:56:34.924635672Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":16,"history_lines":2,"events_offset":26,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:56:35.047221057Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:56:49.924320453Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":18,"history_lines":1,"events_offset":28,"events_lines":1,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:56:50.04108346Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:57:04.924977488Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":19,"history_lines":1,"events_offset":29,"events_lines":3,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:57:05.033285291Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:57:19.92419735Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":20,"history_lines":1,"events_offset":32,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:57:20.086844688Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:57:34.924530245Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":21,"history_lines":2,"events_offset":34,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:57:35.048094586Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:57:49.924278933Z","level":"INFO","msg":"filestream: sending request","total_files":2,"events_offset":36,"events_lines":2,"console_offset":5,"console_lines":1}
+{"time":"2026-08-12T17:57:50.053408952Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:58:04.924247294Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":23,"history_lines":2,"events_offset":38,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:58:05.058183944Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:58:19.924267924Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":25,"history_lines":2,"events_offset":40,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:58:20.079607681Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:58:34.92417317Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":27,"history_lines":1,"events_offset":42,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:58:35.039228745Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:58:49.92475203Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":28,"history_lines":1,"events_offset":44,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:58:50.038320311Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:59:04.924781147Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":29,"history_lines":1,"events_offset":46,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:59:05.042846096Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:59:19.924517427Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":30,"history_lines":2,"events_offset":48,"events_lines":2,"console_offset":2,"console_lines":1}
+{"time":"2026-08-12T17:59:20.048847481Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:59:34.924142827Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":32,"history_lines":1,"events_offset":50,"events_lines":2,"console_offset":5,"console_lines":1}
+{"time":"2026-08-12T17:59:35.090795711Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T17:59:49.924860357Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":33,"history_lines":1,"events_offset":52,"events_lines":1,"console_offset":13,"console_lines":25}
+{"time":"2026-08-12T17:59:50.059568428Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:00:04.924609165Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":34,"history_lines":2,"events_offset":53,"events_lines":3,"console_offset":38,"console_lines":2}
+{"time":"2026-08-12T18:00:05.110076024Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:00:19.925065733Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":36,"history_lines":1,"events_offset":56,"events_lines":2,"console_offset":40,"console_lines":1}
+{"time":"2026-08-12T18:00:20.032512976Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:00:34.924060721Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":58,"events_lines":2}
+{"time":"2026-08-12T18:00:35.038541884Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:00:49.924605Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":37,"history_lines":3,"events_offset":60,"events_lines":2,"console_offset":41,"console_lines":3}
+{"time":"2026-08-12T18:00:50.048539238Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:01:04.924584748Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":40,"history_lines":1,"events_offset":62,"events_lines":2,"console_offset":44,"console_lines":1}
+{"time":"2026-08-12T18:01:05.033829972Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:01:19.924611073Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":41,"history_lines":1,"events_offset":64,"events_lines":2,"console_offset":45,"console_lines":1}
+{"time":"2026-08-12T18:01:20.029593042Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:01:34.925040006Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":42,"history_lines":1,"events_offset":66,"events_lines":2,"console_offset":46,"console_lines":1}
+{"time":"2026-08-12T18:01:35.064189504Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:01:49.924458325Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":43,"history_lines":1,"events_offset":68,"events_lines":2,"console_offset":47,"console_lines":1}
+{"time":"2026-08-12T18:01:50.058784995Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:02:04.925163434Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":44,"history_lines":2,"events_offset":70,"events_lines":2,"console_offset":48,"console_lines":2}
+{"time":"2026-08-12T18:02:05.064760096Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:02:19.924630965Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":46,"history_lines":1,"events_offset":72,"events_lines":2,"console_offset":50,"console_lines":1}
+{"time":"2026-08-12T18:02:20.022710012Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:02:34.924969759Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":47,"history_lines":1,"events_offset":74,"events_lines":2,"console_offset":51,"console_lines":1}
+{"time":"2026-08-12T18:02:35.03681233Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:02:49.924786892Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":48,"history_lines":2,"events_offset":76,"events_lines":1,"console_offset":52,"console_lines":2}
+{"time":"2026-08-12T18:02:50.065566507Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:03:04.924790646Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":50,"history_lines":1,"events_offset":77,"events_lines":3,"console_offset":54,"console_lines":1}
+{"time":"2026-08-12T18:03:05.098703149Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:03:19.924756316Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":80,"events_lines":2}
+{"time":"2026-08-12T18:03:20.074096242Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:03:34.924448572Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":51,"history_lines":2,"events_offset":82,"events_lines":2,"console_offset":55,"console_lines":2}
+{"time":"2026-08-12T18:03:35.063594897Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:03:49.924833484Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":53,"history_lines":2,"events_offset":84,"events_lines":2,"console_offset":57,"console_lines":2}
+{"time":"2026-08-12T18:03:50.05288207Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:04:04.924840703Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":55,"history_lines":1,"events_offset":86,"events_lines":2,"console_offset":59,"console_lines":1}
+{"time":"2026-08-12T18:04:05.096087137Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:04:19.925174248Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":56,"history_lines":1,"events_offset":88,"events_lines":2,"console_offset":60,"console_lines":1}
+{"time":"2026-08-12T18:04:20.072628681Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:04:34.924442094Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":57,"history_lines":1,"events_offset":90,"events_lines":2,"console_offset":61,"console_lines":1}
+{"time":"2026-08-12T18:04:35.039611389Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:04:49.924855584Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":58,"history_lines":1,"events_offset":92,"events_lines":2,"console_offset":62,"console_lines":1}
+{"time":"2026-08-12T18:04:50.072800406Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:05:04.924238971Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":59,"history_lines":2,"events_offset":94,"events_lines":2,"console_offset":63,"console_lines":2}
+{"time":"2026-08-12T18:05:05.097361275Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:05:19.924462456Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":61,"history_lines":1,"events_offset":96,"events_lines":2,"console_offset":65,"console_lines":1}
+{"time":"2026-08-12T18:05:20.029527048Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:05:34.924329692Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":62,"history_lines":1,"events_offset":98,"events_lines":2,"console_offset":66,"console_lines":1}
+{"time":"2026-08-12T18:05:35.02759865Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:05:49.924162531Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":63,"history_lines":2,"events_offset":100,"events_lines":2,"console_offset":67,"console_lines":2}
+{"time":"2026-08-12T18:05:50.0541957Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:06:04.924927099Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":102,"events_lines":2}
+{"time":"2026-08-12T18:06:05.066969956Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:06:19.924165076Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":65,"history_lines":2,"events_offset":104,"events_lines":2,"console_offset":69,"console_lines":2}
+{"time":"2026-08-12T18:06:20.04066715Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:06:34.924283893Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":67,"history_lines":2,"events_offset":106,"events_lines":2,"console_offset":71,"console_lines":2}
+{"time":"2026-08-12T18:06:35.046479676Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:06:49.924399809Z","level":"INFO","msg":"filestream: sending request","total_files":1,"events_offset":108,"events_lines":2}
+{"time":"2026-08-12T18:06:50.042922784Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:07:04.924708661Z","level":"INFO","msg":"filestream: sending request","total_files":4,"history_offset":69,"history_lines":2,"events_offset":110,"events_lines":2,"console_offset":73,"console_lines":7}
+{"time":"2026-08-12T18:07:05.102867255Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:07:12.061660338Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-08-12T18:07:12.061928891Z","level":"INFO","msg":"filestream: sending request","total_files":3,"history_offset":71,"history_lines":1,"console_offset":79,"console_lines":6,"uploaded_len":3,"complete":true,"exit_code":0}
+{"time":"2026-08-12T18:07:12.176355657Z","level":"INFO","msg":"filestream: request sent","status":"200 OK"}
+{"time":"2026-08-12T18:07:12.17743319Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-08-12T18:07:12.180636501Z","level":"INFO","msg":"stream: finishing up"}
+{"time":"2026-08-12T18:07:12.180651797Z","level":"INFO","msg":"handler: closed"}
+{"time":"2026-08-12T18:07:12.180758203Z","level":"INFO","msg":"sender: closed"}
+{"time":"2026-08-12T18:07:12.180766266Z","level":"INFO","msg":"stream: all finished"}

zain/Activation/wandb/run-20260812_175304-hrggki9k/logs/debug.log

@@ -0,0 +1,28 @@
+2026-08-12 17:53:04,075 INFO    MainThread:1371664 [wandb_setup.py:_flush():81] Current SDK version is 0.28.1
+2026-08-12 17:53:04,075 INFO    MainThread:1371664 [wandb_setup.py:_flush():81] Configure stats pid to 1371664
+2026-08-12 17:53:04,075 INFO    MainThread:1371664 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-08-12 17:53:04,075 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():729] Logging user logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_175304-hrggki9k/logs/debug.log
+2026-08-12 17:53:04,075 INFO    MainThread:1371664 [wandb_init.py:setup_run_log_directory():730] Logging internal logs to /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/wandb/run-20260812_175304-hrggki9k/logs/debug-internal.log
+2026-08-12 17:53:04,075 INFO    MainThread:1371664 [wandb_init.py:init():772] calling init triggers
+2026-08-12 17:53:04,075 INFO    MainThread:1371664 [wandb_init.py:init():777] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-08-12 17:53:04,075 INFO    MainThread:1371664 [wandb_init.py:init():820] starting backend
+2026-08-12 17:53:04,075 INFO    MainThread:1371664 [wandb_init.py:init():826] Connected to an existing wandb-core service via WANDB_SERVICE
+2026-08-12 17:53:04,075 INFO    MainThread:1371664 [wandb_init.py:init():835] sending inform_init request
+2026-08-12 17:53:04,339 INFO    MainThread:1371664 [wandb_init.py:init():840] backend started and connected
+2026-08-12 17:53:04,343 INFO    MainThread:1371664 [wandb_init.py:init():910] updated telemetry
+2026-08-12 17:53:04,349 INFO    MainThread:1371664 [wandb_init.py:init():933] communicating run to backend with 90.0 second timeout
+2026-08-12 17:53:04,663 INFO    MainThread:1371664 [wandb_init.py:init():978] starting run threads in backend
+2026-08-12 17:53:04,737 INFO    MainThread:1371664 [wandb_run.py:_console_start():2621] atexit reg
+2026-08-12 17:53:04,737 INFO    MainThread:1371664 [wandb_run.py:_redirect():2471] redirect: wrap_raw
+2026-08-12 17:53:04,737 INFO    MainThread:1371664 [wandb_run.py:_redirect():2540] Wrapping output streams.
+2026-08-12 17:53:04,737 INFO    MainThread:1371664 [wandb_run.py:_redirect():2563] Redirects installed.
+2026-08-12 17:53:04,740 INFO    MainThread:1371664 [wandb_init.py:init():1016] run started, returning control to user process
+2026-08-12 17:53:04,741 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb None None {'transformers_version': '5.16.0.dev0', 'architectures': None, 'output_hidden_states': False, 'return_dict': True, 'dtype': None, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'problem_type': None, 'vocab_size': 4096, 'hidden_size': 128, 'intermediate_size': 256, 'num_hidden_layers': 150, 'num_attention_heads': 4, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'max_position_embeddings': 512, 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'pad_token_id': 0, 'bos_token_id': 1, 'eos_token_id': 2, 'pretraining_tp': 1, 'tie_word_embeddings': True, 'rope_parameters': {'rope_theta': 10000.0, 'rope_type': 'default'}, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 32, '_name_or_path': '', 'tokenizer_name': 'w-ahmad/tiny-stories-tokenizer', 'mlp_type': 'glu', 'activation': 'silu', 'waleed_beta': 10.0, 'model_type': 'tiny_llama', 'output_attentions': False, 'output_dir': 'out/glu-silu-150L_run', 'per_device_train_batch_size': 64, 'num_train_epochs': 1, 'max_steps': 1000, 'learning_rate': 0.005, 'lr_scheduler_type': 'constant_with_warmup', 'lr_scheduler_kwargs': None, 'warmup_steps': 200, 'optim': 'adamw_torch_fused', 'optim_args': None, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'optim_target_modules': None, 'gradient_accumulation_steps': 1, 'average_tokens_across_devices': True, 'max_grad_norm': 0.0, 'label_smoothing_factor': 0.0, 'bf16': True, 'fp16': False, 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'use_liger_kernel': False, 'liger_kernel_config': None, 'neftune_noise_alpha': None, 'torch_empty_cache_steps': None, 'auto_find_batch_size': False, 'logging_strategy': 'steps', 'logging_steps': 20, 'logging_first_step': False, 'log_on_each_node': True, 'logging_nan_inf_filter': True, 'include_num_input_tokens_seen': 'no', 'log_level': 'passive', 'log_level_replica': 'warning', 'disable_tqdm': False, 'report_to': ['wandb'], 'run_name': 'LM-glu-silu-150L-25.1M-20260812-175302', 'project': 'huggingface', 'trackio_space_id': None, 'trackio_bucket_id': None, 'trackio_static_space_id': None, 'eval_strategy': 'steps', 'eval_steps': 50, 'eval_delay': 0, 'per_device_eval_batch_size': 128, 'prediction_loss_only': False, 'eval_on_start': False, 'eval_do_concat_batches': True, 'eval_use_gather_object': False, 'eval_accumulation_steps': None, 'include_for_metrics': [], 'batch_eval_metrics': False, 'save_only_model': False, 'save_strategy': 'steps', 'save_steps': 1000, 'save_on_each_node': False, 'save_total_limit': None, 'enable_jit_checkpoint': False, 'push_to_hub': False, 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': None, 'hub_model_id': 'w-ahmad/6L-glu-silu-150L', 'hub_strategy': 'every_save', 'hub_always_push': False, 'hub_revision': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'restore_callback_states_from_checkpoint': False, 'full_determinism': False, 'seed': 42, 'data_seed': 42, 'use_cpu': False, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'parallelism_config': None, 'dataloader_drop_last': False, 'dataloader_num_workers': 0, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'dataloader_prefetch_factor': None, 'dataloader_multiprocessing_context': None, 'dataloader_in_order': True, 'remove_unused_columns': False, 'label_names': None, 'train_sampling_strategy': 'random', 'length_column_name': 'length', 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'ddp_static_graph': None, 'ddp_backend': None, 'ddp_timeout': 1800, 'fsdp': None, 'fsdp_config': None, 'deepspeed': None, 'debug': [], 'skip_memory_metrics': True, 'do_train': False, 'do_eval': True, 'do_predict': False, 'resume_from_checkpoint': None, 'local_rank': -1}
+2026-08-12 17:53:04,747 INFO    MainThread:1371664 [wandb_config.py:__setitem__():155] [no run ID] config set model/num_parameters = 25138816 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x14c70c726090>>
+2026-08-12 17:53:04,747 INFO    MainThread:1371664 [wandb_run.py:_config_callback():1346] config_cb model/num_parameters 25138816 None
+2026-08-12 18:07:11,666 INFO    MainThread:1371664 [wandb_run.py:_finish():2383] finishing run deepnevro-deepnevro/research2/hrggki9k
+2026-08-12 18:07:11,666 INFO    MainThread:1371664 [wandb_run.py:_atexit_cleanup():2588] got exitcode: 0
+2026-08-12 18:07:11,667 INFO    MainThread:1371664 [wandb_run.py:_restore():2570] restore
+2026-08-12 18:07:11,667 INFO    MainThread:1371664 [wandb_run.py:_restore():2576] restore done
+2026-08-12 18:07:12,180 INFO    MainThread:1371664 [wandb_run.py:_footer_sync_info():3993] logging synced files

zain/Activation/wandb/run-20260812_175304-hrggki9k/run-hrggki9k.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c25d3acdbd13f0cb7e6a536d2b4dca836e3453ebcbdd4b48c5a3f70e00b85f94
+size 929598

zain/Activation/wandb/run-20260812_180713-utqzlg67/files/config.yaml

@@ -0,0 +1,445 @@
+_name_or_path:
+    value: ""
+_wandb:
+    value:
+        cli_version: 0.28.1
+        e:
+            brca6pc78labe08rkhb4qgqvs3b0yaeu:
+                args:
+                    - --config
+                    - /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/configs/baseline150L.yaml
+                    - --variants
+                    - glu-silu
+                    - glu-relu
+                    - glu-gelu
+                    - glu-sigmoid
+                    - glu-linear
+                    - glu-waleed10
+                    - mlp-waleed10
+                    - glu-silu-waleed10
+                    - mlp-silu-waleed10
+                    - glu-situglu
+                    - glu-waleed
+                    - glu-situglu_low
+                    - glu-waleedglu_low
+                codePath: sweep.py
+                codePathLocal: sweep.py
+                cpu_count: 112
+                cpu_count_logical: 224
+                cudaVersion: "12.4"
+                disk:
+                    /:
+                        total: "1560765693952"
+                        used: "716176330752"
+                email: deepnevro@gmail.com
+                executable: /mnt/data/zainulabideen/zain-exp/notebooks/my_env/bin/python
+                git:
+                    commit: c53713bcba08d1849136dad91288c4b82040634e
+                    remote: https://github.com/w-ahmad1a10/Activation.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-39c684a5-fde6-83d7-1663-0859795881ae
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-68012e5a-38b6-b643-0ca6-62fb66720bf3
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-132944c4-b689-2b5f-89a4-d730401677ab
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-2df386cc-6d26-d0e2-7a2d-a057b0d95864
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bfa16575-1d94-1aa2-4537-2c93433f42ef
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-bc6c3e3c-9b90-09ca-c034-774961847c54
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-00a441e1-7c95-e7d6-4c35-43d6b291aea9
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-1c4d29a2-4647-6fce-d8fc-0c5ecfbbd6ea
+                host: deeplens-k3s-node1
+                memory:
+                    total: "2164089937920"
+                os: Linux-5.15.0-126-generic-x86_64-with-glibc2.35
+                program: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation/sweep.py
+                python: CPython 3.11.15
+                root: /mnt/data/zainulabideen/zain-exp/notebooks/zain/Activation
+                startedAt: "2026-08-12T18:07:13.410106Z"
+                writerId: brca6pc78labe08rkhb4qgqvs3b0yaeu
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.11.15
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "2":
+                - 1
+                - 5
+                - 11
+                - 41
+                - 49
+                - 51
+                - 53
+                - 71
+            "3":
+                - 2
+                - 7
+                - 13
+                - 19
+                - 62
+                - 66
+            "4": 3.11.15
+            "5": 0.28.1
+            "6": 5.16.0.dev0
+            "9":
+                "1": transformers_trainer
+            "12": 0.28.1
+            "13": linux-x86_64
+accelerator_config:
+    value:
+        dispatch_batches: null
+        even_batches: true
+        gradient_accumulation_kwargs: null
+        non_blocking: false
+        split_batches: false
+        use_seedable_sampler: true
+activation:
+    value: relu
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+architectures:
+    value: null
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+average_tokens_across_devices:
+    value: true
+batch_eval_metrics:
+    value: false
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: 1
+chunk_size_feed_forward:
+    value: 0
+data_seed:
+    value: 42
+dataloader_drop_last:
+    value: false
+dataloader_in_order:
+    value: true
+dataloader_multiprocessing_context:
+    value: null
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+dataloader_prefetch_factor:
+    value: null
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_static_graph:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+deepspeed:
+    value: null
+disable_tqdm:
+    value: false
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_train:
+    value: false
+dtype:
+    value: null
+enable_jit_checkpoint:
+    value: false
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_do_concat_batches:
+    value: true
+eval_on_start:
+    value: false
+eval_steps:
+    value: 50
+eval_strategy:
+    value: steps
+eval_use_gather_object:
+    value: false
+fp16:
+    value: false
+fp16_full_eval:
+    value: false
+fsdp:
+    value: null
+fsdp_config:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: false
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+head_dim:
+    value: 32
+hidden_act:
+    value: silu
+hidden_size:
+    value: 128
+hub_always_push:
+    value: false
+hub_model_id:
+    value: w-ahmad/6L-glu-relu-150L
+hub_private_repo:
+    value: null
+hub_revision:
+    value: null
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_for_metrics:
+    value: []
+include_num_input_tokens_seen:
+    value: "no"
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 256
+is_encoder_decoder:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.005
+length_column_name:
+    value: length
+liger_kernel_config:
+    value: null
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: -1
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 20
+logging_strategy:
+    value: steps
+lr_scheduler_kwargs:
+    value: null
+lr_scheduler_type:
+    value: constant_with_warmup
+max_grad_norm:
+    value: 0
+max_position_embeddings:
+    value: 512
+max_steps:
+    value: 1000
+metric_for_best_model:
+    value: null
+mlp_bias:
+    value: false
+mlp_type:
+    value: glu
+model/num_parameters:
+    value: 25138816
+model_type:
+    value: tiny_llama
+neftune_noise_alpha:
+    value: null
+num_attention_heads:
+    value: 4
+num_hidden_layers:
+    value: 150
+num_key_value_heads:
+    value: 4
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch_fused
+optim_args:
+    value: null
+optim_target_modules:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: out/glu-relu-150L_run
+output_hidden_states:
+    value: false
+pad_token_id:
+    value: 0
+parallelism_config:
+    value: null
+per_device_eval_batch_size:
+    value: 128
+per_device_train_batch_size:
+    value: 64
+prediction_loss_only:
+    value: false
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+project:
+    value: huggingface
+push_to_hub:
+    value: false
+remove_unused_columns:
+    value: false
+report_to:
+    value:
+        - wandb
+restore_callback_states_from_checkpoint:
+    value: false
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+rms_norm_eps:
+    value: 1e-06
+rope_parameters:
+    value:
+        rope_theta: 10000
+        rope_type: default
+run_name:
+    value: LM-glu-relu-150L-25.1M-20260812-180712
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_steps:
+    value: 1000
+save_strategy:
+    value: steps
+save_total_limit:
+    value: null
+seed:
+    value: 42
+skip_memory_metrics:
+    value: true
+tf32:
+    value: null
+tie_word_embeddings:
+    value: true
+tokenizer_name:
+    value: w-ahmad/tiny-stories-tokenizer
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_empty_cache_steps:
+    value: null
+trackio_bucket_id:
+    value: null
+trackio_space_id:
+    value: null
+trackio_static_space_id:
+    value: null
+train_sampling_strategy:
+    value: random
+transformers_version:
+    value: 5.16.0.dev0
+use_cache:
+    value: false
+use_cpu:
+    value: false
+use_liger_kernel:
+    value: false
+vocab_size:
+    value: 4096
+waleed_beta:
+    value: 10
+warmup_steps:
+    value: 200
+weight_decay:
+    value: 0