Add files using upload-large-folder tool

fla/models/abc/configuration_abc.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+from typing import Dict, Optional
+
+from transformers.configuration_utils import PretrainedConfig
+
+
+class ABCConfig(PretrainedConfig):
+
+    model_type = 'abc'
+    keys_to_ignore_at_inference = ['past_key_values']
+
+    def __init__(
+        self,
+        hidden_size: int = 2048,
+        gate_low_rank_dim: int = 16,
+        clamp_min: float = -32,
+        clamp_max: float = 32,
+        hidden_ratio: Optional[int] = 4,
+        intermediate_size: Optional[int] = None,
+        num_hidden_layers: int = 24,
+        num_heads: int = 4,
+        num_slots: Optional[int] = 64,
+        use_short_conv: bool = False,
+        conv_size: int = 4,
+        exapnd_k: float = 0.5,
+        exapnd_v: float = 1,
+        hidden_act: str = "swish",
+        max_position_embeddings: int = 2048,
+        elementwise_affine: Optional[bool] = True,
+        norm_eps: float = 1e-6,
+        use_rope: bool = True,
+        attn: Optional[Dict] = None,
+        use_cache: bool = True,
+        pad_token_id: int = None,
+        bos_token_id: int = 1,
+        eos_token_id: int = 2,
+        tie_word_embeddings: bool = False,
+        initializer_range: float = 0.006,
+        fuse_norm: bool = True,
+        fuse_swiglu: bool = True,
+        fuse_cross_entropy: bool = True,
+        vocab_size: int = 32000,
+        **kwargs
+    ):
+        self.hidden_size = hidden_size
+        self.gate_low_rank_dim = gate_low_rank_dim
+        self.clamp_min = clamp_min
+        self.clamp_max = clamp_max
+        self.hidden_ratio = hidden_ratio
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_heads = num_heads
+        self.num_slots = num_slots
+        self.use_short_conv = use_short_conv
+        self.conv_size = conv_size
+        self.expand_k = exapnd_k
+        self.expand_v = exapnd_v
+        self.hidden_act = hidden_act
+        self.max_position_embeddings = max_position_embeddings
+        self.elementwise_affine = elementwise_affine
+        self.norm_eps = norm_eps
+        self.use_rope = use_rope
+        self.attn = attn
+        self.use_cache = use_cache
+        self.initializer_range = initializer_range
+
+        self.fuse_norm = fuse_norm
+        self.fuse_swiglu = fuse_swiglu
+        self.fuse_cross_entropy = fuse_cross_entropy
+        self.vocab_size = vocab_size
+
+        if attn is not None:
+            if not isinstance(attn, Dict):
+                raise ValueError("attn must be a dictionary")
+            if 'layers' not in attn:
+                raise ValueError("Layer indices must be provided to initialize hybrid attention layers")
+            if 'num_heads' not in attn:
+                raise ValueError("Number of heads must be provided to initialize hybrid attention layers")
+            attn['num_kv_heads'] = attn.get('num_kv_heads', attn['num_heads'])
+            attn['qkv_bias'] = attn.get('qkv_bias', False)
+            attn['window_size'] = attn.get('window_size', None)
+            attn['rope_theta'] = attn.get('rope_theta', 10000.)
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

fla/models/gla/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+from transformers import AutoConfig, AutoModel, AutoModelForCausalLM
+
+from fla.models.gla.configuration_gla import GLAConfig
+from fla.models.gla.modeling_gla import GLAForCausalLM, GLAModel
+
+AutoConfig.register(GLAConfig.model_type, GLAConfig)
+AutoModel.register(GLAConfig, GLAModel)
+AutoModelForCausalLM.register(GLAConfig, GLAForCausalLM)
+
+
+__all__ = ['GLAConfig', 'GLAForCausalLM', 'GLAModel']

fla/models/hgrn/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+from transformers import AutoConfig, AutoModel, AutoModelForCausalLM
+
+from fla.models.hgrn.configuration_hgrn import HGRNConfig
+from fla.models.hgrn.modeling_hgrn import HGRNForCausalLM, HGRNModel
+
+AutoConfig.register(HGRNConfig.model_type, HGRNConfig)
+AutoModel.register(HGRNConfig, HGRNModel)
+AutoModelForCausalLM.register(HGRNConfig, HGRNForCausalLM)
+
+
+__all__ = ['HGRNConfig', 'HGRNForCausalLM', 'HGRNModel']

fla/models/hgrn/configuration_hgrn.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+
+from typing import Dict, Optional
+
+from transformers.configuration_utils import PretrainedConfig
+
+
+class HGRNConfig(PretrainedConfig):
+
+    model_type = 'hgrn'
+    keys_to_ignore_at_inference = ['past_key_values']
+
+    def __init__(
+        self,
+        attn_mode: str = "fused_recurrent",
+        hidden_size: int = 2048,
+        num_hidden_layers: int = 24,
+        expand_ratio: Optional[int] = 1,
+        use_short_conv: bool = False,
+        conv_size: int = 4,
+        use_lower_bound: bool = True,
+        max_position_embeddings: int = 2048,
+        hidden_ratio: Optional[int] = 4,
+        intermediate_size: Optional[int] = None,
+        hidden_act: str = "swish",
+        elementwise_affine: Optional[bool] = True,
+        norm_eps: float = 1e-6,
+        attn: Optional[Dict] = None,
+        use_cache: bool = True,
+        pad_token_id: int = None,
+        bos_token_id: int = 1,
+        eos_token_id: int = 2,
+        tie_word_embeddings: bool = False,
+        initializer_range: float = 0.006,
+        fuse_norm: bool = True,
+        fuse_swiglu: bool = True,
+        fuse_cross_entropy: bool = True,
+        vocab_size: int = 32000,
+        **kwargs
+    ):
+        self.attn_mode = attn_mode
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.expand_ratio = expand_ratio
+        self.use_short_conv = use_short_conv
+        self.conv_size = conv_size
+        self.use_lower_bound = use_lower_bound
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_ratio = hidden_ratio
+        self.intermediate_size = intermediate_size
+        self.elementwise_affine = elementwise_affine
+        self.attn = attn
+        self.norm_eps = norm_eps
+        self.hidden_act = hidden_act
+        self.use_cache = use_cache
+        self.initializer_range = initializer_range
+
+        self.fuse_norm = fuse_norm
+        self.fuse_swiglu = fuse_swiglu
+        self.fuse_cross_entropy = fuse_cross_entropy
+        self.vocab_size = vocab_size
+
+        if attn is not None:
+            if not isinstance(attn, Dict):
+                raise ValueError("attn must be a dictionary")
+            if 'layers' not in attn:
+                raise ValueError("Layer indices must be provided to initialize hybrid attention layers")
+            if 'num_heads' not in attn:
+                raise ValueError("Number of heads must be provided to initialize hybrid attention layers")
+            attn['num_kv_heads'] = attn.get('num_kv_heads', attn['num_heads'])
+            attn['qkv_bias'] = attn.get('qkv_bias', False)
+            attn['window_size'] = attn.get('window_size', None)
+            attn['rope_theta'] = attn.get('rope_theta', 10000.)
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

fla/models/samba/__init__.py

@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+
+from transformers import AutoConfig, AutoModel, AutoModelForCausalLM
+
+from fla.models.samba.configuration_samba import SambaConfig
+from fla.models.samba.modeling_samba import SambaBlock, SambaForCausalLM, SambaModel
+
+AutoConfig.register(SambaConfig.model_type, SambaConfig, True)
+AutoModel.register(SambaConfig, SambaModel, True)
+AutoModelForCausalLM.register(SambaConfig, SambaForCausalLM, True)
+
+
+__all__ = ['SambaConfig', 'SambaForCausalLM', 'SambaModel', 'SambaBlock']

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2,
+  "transformers_version": "4.51.3"
+}

tb/20251231-0837/wandb/debug-internal.log

@@ -0,0 +1,21 @@
+{"time":"2025-12-31T08:37:38.665057765Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-31T08:37:38.988512957Z","level":"INFO","msg":"stream: created new stream","id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2025-12-31T08:37:38.988613654Z","level":"INFO","msg":"handler: started","stream_id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2025-12-31T08:37:38.988714345Z","level":"INFO","msg":"stream: started","id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2025-12-31T08:37:38.988738297Z","level":"INFO","msg":"writer: started","stream_id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2025-12-31T08:37:38.988753655Z","level":"INFO","msg":"sender: started","stream_id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2025-12-31T11:49:59.759567501Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/file_stream\": dial tcp 35.186.228.49:443: connect: connection refused"}
+{"time":"2025-12-31T13:32:20.799679052Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":26941}
+{"time":"2025-12-31T13:32:20.80069231Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":2}
+{"time":"2025-12-31T21:52:16.560064623Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/file_stream\": dial tcp 35.186.228.49:443: connect: connection refused"}
+{"time":"2025-12-31T23:40:07.918310102Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/file_stream\": dial tcp: lookup api.wandb.ai on 10.43.0.10:53: read udp 10.42.35.131:44189->10.43.0.10:53: i/o timeout"}
+{"time":"2025-12-31T23:47:01.610252645Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/file_stream\": dial tcp 35.186.228.49:443: connect: connection refused"}
+{"time":"2026-01-01T08:38:59.64892198Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/file_stream\": dial tcp 35.186.228.49:443: connect: connection refused"}
+{"time":"2026-01-01T09:06:12.449377603Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":129853}
+{"time":"2026-01-01T09:06:12.451188339Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
+{"time":"2026-01-01T09:07:50.503866067Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-01-01T09:07:50.688587533Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-01-01T09:07:50.69317498Z","level":"INFO","msg":"stream: closing","id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2026-01-01T09:07:50.693203953Z","level":"INFO","msg":"handler: closed","stream_id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2026-01-01T09:07:50.693272578Z","level":"INFO","msg":"sender: closed","stream_id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2026-01-01T09:07:50.693279066Z","level":"INFO","msg":"stream: closed","id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}

tb/20251231-0837/wandb/debug.log

@@ -0,0 +1,26 @@
+2025-12-31 08:37:38,391 INFO    MainThread:3901702 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_setup.py:_flush():80] Configure stats pid to 3901702
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_setup.py:_flush():80] Loading settings from /root/flame/wandb/settings
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_init.py:setup_run_log_directory():714] Logging user logs to exp/dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine/tb/20251231-0837/wandb/run-20251231_083738--dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/logs/debug.log
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to exp/dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine/tb/20251231-0837/wandb/run-20251231_083738--dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/logs/debug-internal.log
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_init.py:init():841] calling init triggers
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_init.py:init():889] starting backend
+2025-12-31 08:37:38,658 INFO    MainThread:3901702 [wandb_init.py:init():892] sending inform_init request
+2025-12-31 08:37:38,663 INFO    MainThread:3901702 [wandb_init.py:init():900] backend started and connected
+2025-12-31 08:37:38,666 INFO    MainThread:3901702 [wandb_init.py:init():970] updated telemetry
+2025-12-31 08:37:38,673 INFO    MainThread:3901702 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-31 08:37:39,397 INFO    MainThread:3901702 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-31 08:37:39,496 INFO    MainThread:3901702 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-31 08:37:39,496 INFO    MainThread:3901702 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-31 08:37:39,496 INFO    MainThread:3901702 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-31 08:37:39,496 INFO    MainThread:3901702 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-31 08:37:39,499 INFO    MainThread:3901702 [wandb_init.py:init():1081] run started, returning control to user process
+2026-01-01 09:07:50,099 INFO    MainThread:3901702 [wandb_run.py:_finish():2287] finishing run zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834
+2026-01-01 09:07:50,099 INFO    MainThread:3901702 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0
+2026-01-01 09:07:50,100 INFO    MainThread:3901702 [wandb_run.py:_restore():2468] restore
+2026-01-01 09:07:50,100 INFO    MainThread:3901702 [wandb_run.py:_restore():2474] restore done
+2026-01-01 09:07:50,692 INFO    MainThread:3901702 [wandb_run.py:_footer_sync_info():3862] logging synced files

tb/20251231-0837/wandb/run-20251231_083738--dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/files/requirements.txt

@@ -0,0 +1,169 @@
+flame==0.1.0
+fsspec==2025.10.0
+aiohappyeyeballs==2.6.1
+ipykernel==7.1.0
+smmap==5.0.2
+pybind11==3.0.1
+tabulate==0.9.0
+parso==0.8.5
+yarl==1.22.0
+asttokens==3.0.1
+pandas==2.3.3
+xxhash==3.6.0
+pathvalidate==3.3.1
+Werkzeug==3.1.4
+regex==2025.11.3
+inquirerpy==0.3.4
+click==8.3.1
+idna==3.11
+pydantic==2.12.5
+pexpect==4.9.0
+typepy==1.3.4
+certifi==2025.11.12
+wcwidth==0.2.14
+triton==3.2.0
+hf-xet==1.2.0
+joblib==1.5.3
+tqdm==4.67.1
+nvidia-nvtx-cu12==12.4.127
+setuptools==80.9.0
+lxml==6.0.2
+nvidia-cufft-cu12==11.2.1.3
+evaluate==0.4.6
+Markdown==3.10
+chardet==5.2.0
+multiprocess==0.70.18
+tensorboard==2.20.0
+nvidia-nvjitlink-cu12==12.4.127
+flame==0.1.0
+matplotlib-inline==0.2.1
+Cython==3.2.3
+tensorboard-data-server==0.7.2
+nvidia-cusparse-cu12==12.3.1.170
+lm_eval==0.4.9.1
+pure_eval==0.2.3
+protobuf==6.33.2
+DataProperty==1.1.0
+nvidia-cudnn-cu12==9.1.0.70
+accelerate==1.12.0
+psutil==7.1.3
+Jinja2==3.1.6
+scikit-learn==1.8.0
+nvidia-nccl-cu12==2.21.5
+typing_extensions==4.15.0
+pyzmq==27.1.0
+mpmath==1.3.0
+annotated-types==0.7.0
+propcache==0.4.1
+wandb==0.23.1
+requests==2.32.5
+ipython==9.8.0
+more-itertools==10.8.0
+nvidia-cuda-runtime-cu12==12.4.127
+sacrebleu==2.5.1
+httpx==0.28.1
+huggingface-hub==0.36.0
+MarkupSafe==3.0.3
+nvidia-cusolver-cu12==11.6.1.9
+gitdb==4.0.12
+torchdata==0.11.0
+sentry-sdk==2.48.0
+sympy==1.13.1
+safetensors==0.7.0
+httpcore==1.0.9
+portalocker==3.2.0
+attrs==25.4.0
+typing-inspection==0.4.2
+ptyprocess==0.7.0
+nvidia-cublas-cu12==12.4.5.8
+numexpr==2.14.1
+executing==2.2.1
+networkx==3.6.1
+threadpoolctl==3.6.0
+nvidia-cusparselt-cu12==0.6.2
+einops==0.8.1
+zstandard==0.25.0
+comm==0.2.3
+six==1.17.0
+packaging==25.0
+tqdm-multiprocess==0.0.11
+numpy==2.3.5
+colorama==0.4.6
+nvidia-cuda-cupti-cu12==12.4.127
+jupyter_client==8.7.0
+scipy==1.16.3
+tornado==6.5.4
+nltk==3.9.2
+antlr4-python3-runtime==4.11.0
+jupyter_core==5.9.1
+sqlitedict==2.1.0
+tzdata==2025.3
+pytz==2025.2
+Pygments==2.19.2
+python-dotenv==1.2.1
+cmake==4.2.0
+tiktoken==0.12.0
+PyYAML==6.0.3
+datasets==4.4.1
+pillow==12.0.0
+math-verify==0.8.0
+dill==0.4.0
+nvidia-cuda-nvrtc-cu12==12.4.127
+anyio==4.12.0
+prompt_toolkit==3.0.52
+filelock==3.20.1
+jedi==0.19.2
+frozenlist==1.8.0
+tokenizers==0.21.4
+grpcio==1.76.0
+ninja==1.13.0
+mbstrdecoder==1.1.4
+flash-attn==2.7.3
+aiosignal==1.4.0
+tabledata==1.3.4
+h11==0.16.0
+absl-py==2.3.1
+latex2sympy2_extended==1.10.2
+torch==2.6.0
+nest_asyncio==1.6.0
+pip==25.3
+aiohttp==3.13.2
+pfzy==0.3.4
+platformdirs==4.5.1
+wheel==0.45.1
+peft==0.17.0
+debugpy==1.8.19
+ipython_pygments_lexers==1.1.1
+rouge_score==0.1.2
+multidict==6.7.0
+tcolorpy==0.1.7
+nvidia-curand-cu12==10.3.5.147
+pydantic_core==2.41.5
+pytablewriter==1.2.1
+charset-normalizer==3.4.4
+transformers==4.51.3
+word2number==1.1
+jsonlines==4.0.0
+stack_data==0.6.3
+urllib3==2.6.2
+decorator==5.2.1
+python-dateutil==2.9.0.post0
+pyarrow==22.0.0
+traitlets==5.14.3
+GitPython==3.1.45
+tomli==2.0.1
+more-itertools==10.3.0
+inflect==7.3.1
+zipp==3.19.2
+jaraco.functools==4.0.1
+autocommand==2.2.2
+jaraco.collections==5.1.0
+platformdirs==4.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+jaraco.text==3.12.1
+typing_extensions==4.12.2
+jaraco.context==5.3.0
+typeguard==4.3.0
+packaging==24.2
+wheel==0.45.1

tb/20251231-0837/wandb/run-20251231_083738--dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/files/wandb-summary.json

@@ -0,0 +1 @@
+{"loss_metrics/global_avg_loss":3.5242035388946533,"throughput(tps)":29851.614107024114,"time_metrics/end_to_end(s)":2.1953921742737292,"memory/max_active(GiB)":119.60354804992676,"_timestamp":1.767258330571418e+09,"memory/num_ooms":0,"time_metrics/data_loading(%)":0.05893591801712437,"time_metrics/data_loading(s)":0.0012938745319843292,"loss_metrics/global_avg_mtp_loss":2.688770055770874,"memory/max_active(%)":85.82660800240015,"loss_metrics/global_max_loss":3.867431163787842,"tflops":420.1594759014195,"memory/num_alloc_retries":0,"_step":40000,"memory/max_reserved(%)":88.07186436520833,"optimizer/grad_norm":0.7287646532058716,"_runtime":88210.702133884,"loss_metrics/global_avg_ntp_loss":0.8354334831237793,"_wandb":{"runtime":88210},"optimizer/lr":5e-06,"mfu(%)":42.483263488515625,"optimizer/skipped_step":0,"memory/max_reserved(GiB)":122.732421875}

tb/20251231-0837/wandb/run-20251231_083738--dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2025-12-31T08:37:38.477715818Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp9ir1595i/port-3901702.txt","pid":3901702,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-12-31T08:37:38.479587659Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3901702}
+{"time":"2025-12-31T08:37:38.479566978Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3901702-3903170-2736290377/socket","Net":"unix"}}
+{"time":"2025-12-31T08:37:38.65778368Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-12-31T08:37:38.664866914Z","level":"INFO","msg":"handleInformInit: received","streamId":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834","id":"1(@)"}
+{"time":"2025-12-31T08:37:38.988721434Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834","id":"1(@)"}
+{"time":"2026-01-01T09:07:50.693137528Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834","id":"1(@)"}
+{"time":"2026-01-01T09:07:50.693292254Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834","id":"1(@)"}
+{"time":"2026-01-01T09:08:12.419440178Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-01-01T09:08:12.419495994Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-01-01T09:08:12.41950792Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-01-01T09:08:12.419553284Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-01-01T09:08:12.419557333Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-01-01T09:08:12.419563866Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-01-01T09:08:12.419751426Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3901702-3903170-2736290377/socket","Net":"unix"}}
+{"time":"2026-01-01T09:08:12.419802874Z","level":"INFO","msg":"server is closed"}

tb/20251231-0837/wandb/run-20251231_083738--dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/logs/debug-internal.log

@@ -0,0 +1,21 @@
+{"time":"2025-12-31T08:37:38.665057765Z","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
+{"time":"2025-12-31T08:37:38.988512957Z","level":"INFO","msg":"stream: created new stream","id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2025-12-31T08:37:38.988613654Z","level":"INFO","msg":"handler: started","stream_id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2025-12-31T08:37:38.988714345Z","level":"INFO","msg":"stream: started","id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2025-12-31T08:37:38.988738297Z","level":"INFO","msg":"writer: started","stream_id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2025-12-31T08:37:38.988753655Z","level":"INFO","msg":"sender: started","stream_id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2025-12-31T11:49:59.759567501Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/file_stream\": dial tcp 35.186.228.49:443: connect: connection refused"}
+{"time":"2025-12-31T13:32:20.799679052Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":26941}
+{"time":"2025-12-31T13:32:20.80069231Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":2}
+{"time":"2025-12-31T21:52:16.560064623Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/file_stream\": dial tcp 35.186.228.49:443: connect: connection refused"}
+{"time":"2025-12-31T23:40:07.918310102Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/file_stream\": dial tcp: lookup api.wandb.ai on 10.43.0.10:53: read udp 10.42.35.131:44189->10.43.0.10:53: i/o timeout"}
+{"time":"2025-12-31T23:47:01.610252645Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/file_stream\": dial tcp 35.186.228.49:443: connect: connection refused"}
+{"time":"2026-01-01T08:38:59.64892198Z","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/file_stream\": dial tcp 35.186.228.49:443: connect: connection refused"}
+{"time":"2026-01-01T09:06:12.449377603Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":129853}
+{"time":"2026-01-01T09:06:12.451188339Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":1}
+{"time":"2026-01-01T09:07:50.503866067Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-01-01T09:07:50.688587533Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-01-01T09:07:50.69317498Z","level":"INFO","msg":"stream: closing","id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2026-01-01T09:07:50.693203953Z","level":"INFO","msg":"handler: closed","stream_id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2026-01-01T09:07:50.693272578Z","level":"INFO","msg":"sender: closed","stream_id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}
+{"time":"2026-01-01T09:07:50.693279066Z","level":"INFO","msg":"stream: closed","id":"-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834"}

tb/20251231-0837/wandb/run-20251231_083738--dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/logs/debug.log

@@ -0,0 +1,26 @@
+2025-12-31 08:37:38,391 INFO    MainThread:3901702 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_setup.py:_flush():80] Configure stats pid to 3901702
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_setup.py:_flush():80] Loading settings from /root/flame/wandb/settings
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_init.py:setup_run_log_directory():714] Logging user logs to exp/dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine/tb/20251231-0837/wandb/run-20251231_083738--dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/logs/debug.log
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to exp/dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine/tb/20251231-0837/wandb/run-20251231_083738--dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834/logs/debug-internal.log
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_init.py:init():841] calling init triggers
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-12-31 08:37:38,392 INFO    MainThread:3901702 [wandb_init.py:init():889] starting backend
+2025-12-31 08:37:38,658 INFO    MainThread:3901702 [wandb_init.py:init():892] sending inform_init request
+2025-12-31 08:37:38,663 INFO    MainThread:3901702 [wandb_init.py:init():900] backend started and connected
+2025-12-31 08:37:38,666 INFO    MainThread:3901702 [wandb_init.py:init():970] updated telemetry
+2025-12-31 08:37:38,673 INFO    MainThread:3901702 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
+2025-12-31 08:37:39,397 INFO    MainThread:3901702 [wandb_init.py:init():1041] starting run threads in backend
+2025-12-31 08:37:39,496 INFO    MainThread:3901702 [wandb_run.py:_console_start():2521] atexit reg
+2025-12-31 08:37:39,496 INFO    MainThread:3901702 [wandb_run.py:_redirect():2369] redirect: wrap_raw
+2025-12-31 08:37:39,496 INFO    MainThread:3901702 [wandb_run.py:_redirect():2438] Wrapping output streams.
+2025-12-31 08:37:39,496 INFO    MainThread:3901702 [wandb_run.py:_redirect():2461] Redirects installed.
+2025-12-31 08:37:39,499 INFO    MainThread:3901702 [wandb_init.py:init():1081] run started, returning control to user process
+2026-01-01 09:07:50,099 INFO    MainThread:3901702 [wandb_run.py:_finish():2287] finishing run zaydzuhri/fla/-dsmtp.code.1B.batch16.seqlen4096.context4096.warmup400.update1.steps40000.lr5e-5.cosine-202512310834
+2026-01-01 09:07:50,099 INFO    MainThread:3901702 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0
+2026-01-01 09:07:50,100 INFO    MainThread:3901702 [wandb_run.py:_restore():2468] restore
+2026-01-01 09:07:50,100 INFO    MainThread:3901702 [wandb_run.py:_restore():2474] restore done
+2026-01-01 09:07:50,692 INFO    MainThread:3901702 [wandb_run.py:_footer_sync_info():3862] logging synced files