Add 350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621

.gitattributes

@@ -90,3 +90,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 350m/adamp_lr1e_3_b1_0_9_b2_0_98_eps_1e_8_A100_ppl_17_3947_20260416_223206/wandb/offline-run-20260416_223228-93itajip/run-93itajip.wandb filter=lfs diff=lfs merge=lfs -text
 350m/adamw_lr1e_3_b1_0_9_b2_0_99_eps_1e_8_A100_ppl_17_3227_20260416_225515/wandb/offline-run-20260416_225539-oqwr5f9l/run-oqwr5f9l.wandb filter=lfs diff=lfs merge=lfs -text
 350m/adan_lr3e_3_b1_0_9_b2_0_92_b3_0_99_eps_1e_8_A100_ppl_17_2895_20260416_200526/wandb/offline-run-20260416_200602-wk4fdtc0/run-wk4fdtc0.wandb filter=lfs diff=lfs merge=lfs -text
+350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/run-dhgtud9k.wandb filter=lfs diff=lfs merge=lfs -text

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/model_60000/config.json

@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": 1,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 1024,
+  "initializer_range": 0.02,
+  "intermediate_size": 2736,
+  "max_position_embeddings": 2048,
+  "max_sequence_length": 1024,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 16,
+  "pad_token_id": -1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.3",
+  "use_cache": true,
+  "vocab_size": 32000
+}

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/model_60000/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c4f9a1a8aa6a34e444aaab39e5ba9a4bcad711e1318447e9639d0406603b372
+size 735967792

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/model_60000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36799361bd4dabf143e316bf8e029b2be9cfd1eee87c99f22554c02cfb4d4830
+size 653052922

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/model_60000/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a0cdb46285cbb6b4582783d67b41e68bd1248c506a309fb698f7a78cd08052e
+size 736040086

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/model_60000/training_state.json

@@ -0,0 +1,7 @@
+{
+    "global_step": 60000,
+    "update_step": 60000,
+    "tokens_seen": 5999255724,
+    "tokens_seen_before": 5999150160,
+    "update_time": 1.0080578327178955
+}

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/wandb.json

@@ -0,0 +1,3 @@
+{
+    "wandb_id": "dhgtud9k"
+}

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/wandb/debug-internal.log

@@ -0,0 +1,12 @@
+{"time":"2026-04-19T23:47:18.48078063+08:00","level":"INFO","msg":"stream: starting","core version":"0.23.0"}
+{"time":"2026-04-19T23:47:18.612202918+08:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-04-19T23:47:18.612265953+08:00","level":"INFO","msg":"stream: created new stream","id":"dhgtud9k"}
+{"time":"2026-04-19T23:47:18.612302459+08:00","level":"INFO","msg":"handler: started","stream_id":"dhgtud9k"}
+{"time":"2026-04-19T23:47:18.617535039+08:00","level":"INFO","msg":"stream: started","id":"dhgtud9k"}
+{"time":"2026-04-19T23:47:18.617543876+08:00","level":"INFO","msg":"writer: started","stream_id":"dhgtud9k"}
+{"time":"2026-04-19T23:47:18.617549943+08:00","level":"INFO","msg":"sender: started","stream_id":"dhgtud9k"}
+{"time":"2026-04-19T23:47:18.618872114+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-04-20T09:55:26.509801244+08:00","level":"INFO","msg":"stream: closing","id":"dhgtud9k"}
+{"time":"2026-04-20T09:55:26.510476014+08:00","level":"INFO","msg":"handler: closed","stream_id":"dhgtud9k"}
+{"time":"2026-04-20T09:55:26.511606678+08:00","level":"INFO","msg":"sender: closed","stream_id":"dhgtud9k"}
+{"time":"2026-04-20T09:55:26.511618561+08:00","level":"INFO","msg":"stream: closed","id":"dhgtud9k"}

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-04-19 23:47:17,983 INFO    MainThread:111045 [wandb_setup.py:_flush():80] Current SDK version is 0.23.0
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_setup.py:_flush():80] Configure stats pid to 111045
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_setup.py:_flush():80] Loading settings from /mnt/petrelfs/panjiabao/.config/wandb/settings
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_setup.py:_flush():80] Loading settings from /mnt/petrelfs/panjiabao/Optimizer/SAC/wandb/settings
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_init.py:setup_run_log_directory():713] Logging user logs to /mnt/dhwfile/tancheng/panjiabao/Result/SAC_C4/work_dirs/350m/apollo_sweep_lr1e-2_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/logs/debug.log
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_init.py:setup_run_log_directory():714] Logging internal logs to /mnt/dhwfile/tancheng/panjiabao/Result/SAC_C4/work_dirs/350m/apollo_sweep_lr1e-2_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/logs/debug-internal.log
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_init.py:init():840] calling init triggers
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_init.py:init():845] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_init.py:init():888] starting backend
+2026-04-19 23:47:18,397 INFO    MainThread:111045 [wandb_init.py:init():891] sending inform_init request
+2026-04-19 23:47:18,425 INFO    MainThread:111045 [wandb_init.py:init():899] backend started and connected
+2026-04-19 23:47:18,429 INFO    MainThread:111045 [wandb_init.py:init():969] updated telemetry
+2026-04-19 23:47:18,474 INFO    MainThread:111045 [wandb_init.py:init():993] communicating run to backend with 90.0 second timeout
+2026-04-19 23:47:18,620 INFO    MainThread:111045 [wandb_init.py:init():1040] starting run threads in backend
+2026-04-19 23:47:19,015 INFO    MainThread:111045 [wandb_run.py:_console_start():2504] atexit reg
+2026-04-19 23:47:19,015 INFO    MainThread:111045 [wandb_run.py:_redirect():2352] redirect: wrap_raw
+2026-04-19 23:47:19,015 INFO    MainThread:111045 [wandb_run.py:_redirect():2421] Wrapping output streams.
+2026-04-19 23:47:19,015 INFO    MainThread:111045 [wandb_run.py:_redirect():2444] Redirects installed.
+2026-04-19 23:47:19,023 INFO    MainThread:111045 [wandb_init.py:init():1080] run started, returning control to user process
+2026-04-19 23:47:27,035 INFO    MainThread:111045 [wandb_run.py:_config_callback():1385] config_cb None None {'model_config': 'configs/llama_350m.json', 'exp_config': 'exp_v2/configs/llama_350m_apollo.json', 'eval_every': 1000, 'save_every': 60000, 'dtype': 'bfloat16', 'seed': 0, 'compile': True, 'dynamo_suppress_errors': True, 'dynamo_cache_limit': 10000, 'memory_cleanup_frequency': 10000, 'resume_step': None, 'restore_optimizer': False, 'continue_from': None, 'single_gpu': False, 'save_dir': '/mnt/dhwfile/tancheng/panjiabao/Result/SAC_C4/work_dirs/350m/apollo_sweep_lr1e-2_20260419_234621', 'use_hf_model': False, 'workers': 12, 'batch_size': 128, 'gradient_accumulation': 1, 'total_batch_size': 512, 'warmup_steps': 6000, 'num_training_steps': 60000, 'max_train_tokens': None, 'optimizer': 'apollo_adamw', 'max_length': 256, 'scheduler': 'cosine', 'min_lr_ratio': 0.1, 'weight_decay': 0.0, 'grad_clipping': 0.0, 'activation_checkpointing': False, 'data_path': '/mnt/dhwfile/tancheng/panjiabao/dataset/C4/en', 'data_name': 'en', 'tags': None, 'name': 'test', 'project': 'test', 'unset_wandb': False, 'entity': None, 'wandb_dir': '/mnt/dhwfile/tancheng/panjiabao/Result/SAC_C4/work_dirs/350m/apollo_sweep_lr1e-2_20260419_234621', 'beta1': 0.9, 'beta2': 0.99, 'beta3': 0.99, 'eps': 1e-06, 'rank': 256, 'update_proj_gap': 200, 'galore_scale': 1.0, 'proj_type': 'std', 'proj_quant': False, 'proj_bits': 8, 'proj_group_size': 256, 'weight_quant': False, 'weight_bits': 8, 'weight_group_size': 256, 'stochastic_round': False, 'simulation': False, 'cos_threshold': 1, 'gamma_proj': 2, 'queue_size': 5, 'proj': 'random', 'scale_type': 'channel', 'apollo_scale': 1, 'scale_front': False, 'n_clusters': 3, 'scale_update_freq': 500, 'scale_level': '1,0,1,1', 'scale_bound': None, 'metric': 'mean', 'align_grad': False, 'dim': 4096, 'n_heads': 32, 'muon_ns_steps': 5, 'muon_momentum': 0.95, 'nproc_per_node': 4, 'max_lr': 0.01, 'total_params_M': 367.96928, 'dataset': 'c4', 'model': {'vocab_size': 32000, 'max_position_embeddings': 2048, 'hidden_size': 1024, 'intermediate_size': 2736, 'num_hidden_layers': 24, 'num_attention_heads': 16, 'num_key_value_heads': 16, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 10000.0, 'rope_scaling': None, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 64, 'return_dict': True, 'output_hidden_states': False, 'torchscript': False, 'dtype': None, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'architectures': ['LLaMAForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'task_specific_params': None, 'problem_type': None, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 0, 'pad_token_id': -1, 'eos_token_id': 1, 'sep_token_id': None, 'decoder_start_token_id': None, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'num_beam_groups': 1, 'diversity_penalty': 0.0, '_name_or_path': 'configs/llama_350m.json', 'transformers_version': '4.57.3', 'max_sequence_length': 1024, 'model_type': 'llama', 'tf_legacy_loss': False, 'use_bfloat16': False, 'output_attentions': False}, 'world_size': 4, 'device': 'cuda:0'}
+2026-04-20 09:55:26,509 INFO    wandb-AsyncioManager-main:111045 [service_client.py:_forward_responses():80] Reached EOF.
+2026-04-20 09:55:26,510 INFO    wandb-AsyncioManager-main:111045 [mailbox.py:close():137] Closing mailbox, abandoning 0 handles.

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/files/SAC/torchrun_main.py

@@ -0,0 +1,603 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+import os
+import time
+import json
+
+import torch
+import torch.nn as nn
+import torch.distributed as dist
+
+from tqdm import tqdm
+from loguru import logger
+
+import transformers
+
+transformers.logging.set_verbosity_error()
+
+import wandb
+
+from utils.argparse import parse_args
+from utils.setup import getting_svd_cnt, set_seed, setup_model, saving_model_weight, load_model_weight
+from utils.optimizer_factory import setup_optimization
+from utils.eval import evaluate_model
+from utils.dataloader import setup_dataset
+from utils.modeling_llama import LlamaForCausalLM
+from utils.fake_quantization import QLinear
+from utils.quantization import QScaleLinear
+
+
+def main(args):
+    import torch
+    ############ Setup random seed ############
+    set_seed(args)
+
+    ############ Setup DDP environment ############
+    assert "LOCAL_RANK" in os.environ, "torchrun should set LOCAL_RANK"
+    global_rank = int(os.environ["RANK"])
+    local_rank = int(os.environ["LOCAL_RANK"])
+    world_size = int(os.environ["WORLD_SIZE"])
+    torch.cuda.set_device(local_rank)
+
+    logger.info(f"Global rank {global_rank}, local rank {local_rank}, device: {torch.cuda.current_device()}")
+    dist.init_process_group(backend="nccl", rank=global_rank, world_size=world_size)
+
+    logger.info("Process group initialized")
+    device = f"cuda:{local_rank}"
+
+    if global_rank != 0:
+        logger.remove()  # turn off logger
+
+    logger.info(f"Using dist with rank {global_rank} (only rank 0 will log)")
+    logger.info("*" * 40)
+    logger.info(f"Starting training with the arguments")
+    for k, v in vars(args).items():
+        logger.info(f"{k:30} {v}")
+    logger.info("*" * 40)
+
+    ############ Initialize wandb without config (it is passed later) ############
+    if (not args.unset_wandb) and global_rank == 0:
+        if args.entity is None:
+            os.environ['WANDB_MODE'] = 'offline'
+        # Set wandb directory for offline mode
+        wandb_dir = getattr(args, 'wandb_dir', None) if getattr(args, 'wandb_dir', None) is not None else args.save_dir
+        if getattr(args, 'wandb_dir', None) is not None:
+            logger.info(f"Wandb directory set to: {wandb_dir}")
+        wandb.init(project=args.project, name=args.name, entity=args.entity, dir=wandb_dir)
+
+    ############ Setup training data ############
+    if args.total_batch_size is not None:
+        if args.gradient_accumulation is None:
+            assert args.total_batch_size % world_size == 0, "total_batch_size must be divisible by world_size"
+            args.gradient_accumulation = args.total_batch_size // (args.batch_size * world_size)
+            assert args.gradient_accumulation > 0, "gradient_accumulation must be greater than 0"
+
+    assert (
+        args.gradient_accumulation * args.batch_size * world_size == args.total_batch_size
+    ), "gradient_accumulation * batch_size * world_size must be equal to total_batch_size"
+
+    dataloader, tokenizer = setup_dataset(args, global_rank, world_size)
+
+    ############ Initialize model ############
+    model_config, model = setup_model(args)
+    # Ensure model has generation_config (fix for transformers version compatibility)
+    if model.generation_config is None:
+        from transformers import GenerationConfig
+        model.generation_config = GenerationConfig()
+    model.generation_config.pad_token_id = tokenizer.pad_token_id
+
+    ############ Resuming from checkpoints ############
+    global_step = 0
+    update_step = 0
+    beginning_step = 0
+    tokens_seen = 0
+    tokens_seen_before = 0
+
+    # identifying checkpointing
+    if args.continue_from is not None and os.path.exists(args.continue_from):
+        # searching the latest checkpoints
+        checkpoint_path_list = os.listdir(args.continue_from)
+        checkpoint_path_list = [int(x.split("_")[-1]) for x in checkpoint_path_list if x.startswith("model_")]
+        if len(checkpoint_path_list) > 0:
+            logger.info("Find Checkpoints", checkpoint_path_list)
+            beginning_step = max(checkpoint_path_list)
+            if args.resume_step is not None:
+                beginning_step = args.resume_step
+            args.continue_from = os.path.join(args.continue_from, f"model_{beginning_step}")
+            logger.info("Continue from", args.continue_from)
+        else:
+            logger.warning(f"Did not find any checkpoints in {args.continue_from}")
+            args.continue_from = None
+
+    # resuming from checkpointing
+    if args.continue_from is not None:
+        logger.info("*" * 40)
+        logger.info(f"Loading model from {args.continue_from}")
+        checkpoint_path = os.path.join(args.continue_from, "pytorch_model.bin")
+        if os.path.exists(checkpoint_path):
+            load_model_weight(model, checkpoint_path, args)
+            logger.info(f"Model successfully loaded (strict=False policy)")
+        else:
+            # Try safetensors format
+            checkpoint_path = os.path.join(args.continue_from, "model.safetensors")
+            if os.path.exists(checkpoint_path):
+                from safetensors import safe_open
+                tensors = {}
+                with safe_open(checkpoint_path, framework="pt", device=0) as f:
+                    for k in f.keys():
+                        tensors[k] = f.get_tensor(k)
+                        print(k, tensors[k].shape)
+                ret = model.load_state_dict(tensors, strict=False)
+                logger.info(f"Model successfully loaded from safetensors (strict=False policy)", ret)
+            else:
+                logger.warning(f"No model checkpoint found in {args.continue_from}")
+
+        if os.path.exists(os.path.join(args.continue_from, "training_state.json")):
+            logger.info(
+                f"Loading training state like global_step, update_step, and tokens_seen from {args.continue_from}"
+            )
+            with open(os.path.join(args.continue_from, "training_state.json")) as f:
+                _old_state = json.load(f)
+            global_step = _old_state["global_step"]
+            update_step = _old_state["update_step"]
+            tokens_seen = _old_state["tokens_seen"]
+            tokens_seen_before = _old_state["tokens_seen_before"]
+            logger.info(f"global_step       : {global_step}")
+            logger.info(f"update_step       : {update_step}")
+            logger.info(f"tokens_seen       : {tokens_seen}")
+            logger.info(f"tokens_seen_before: {tokens_seen_before}")
+            logger.info(f"Will train for {args.num_training_steps - update_step} update steps")
+        else:
+            logger.warning(f"Did not find training state in {args.continue_from}, global step will start from zero")
+        logger.info("*" * 40)
+
+    ############ Setup model ############
+    if args.dtype in ["bf16", "bfloat16"]:
+        model = model.to(dtype=torch.bfloat16)
+    model = model.to(device=device)
+
+    for _, module in model.named_modules():
+        if isinstance(module, QScaleLinear):
+            weight_device = module.weight.device
+            module.weight.scales = module.weight.scales.to(device=weight_device)
+            module.weight.zeros = module.weight.zeros.to(device=weight_device)
+
+    n_total_params = sum(p.numel() for p in model.parameters())
+    trainable_params = [p for p in model.parameters() if p.requires_grad]
+    trainable_params_int8 = [p for p in model.parameters() if hasattr(p, "group_size")]
+
+    ############ Initialize wandb ############
+    run_config = dict(vars(args))
+    run_config.update(
+        {
+            "max_lr": run_config.pop("lr"),  # rename lr to max_lr to avoid conflicts with scheduler
+            "total_params_M": n_total_params / 1_000_000,
+            "dataset": "c4",
+            "model": model_config.to_dict(),
+            "world_size": world_size,
+            "device": str(device),
+        }
+    )
+
+    if global_rank == 0:
+        if not args.unset_wandb:
+            wandb.config.update(run_config, allow_val_change=True)
+            wandb.save(os.path.abspath(__file__), policy="now")  # save current script
+        # fix tqdm visual length to 80 so that the progress bar
+        # doesn't jump around when changing from external display to laptop
+        pbar = tqdm(total=args.num_training_steps - update_step, desc="Update steps", ncols=80)
+
+    ############ Initialize optimization ############
+    if "galore" in args.optimizer.lower():
+        # make parameters with "rank" to a single group, if param_name has "mlp" or "attn"
+        lowrank_params = []
+        target_modules_list = ["attn", "mlp"]
+        for module_name, module in model.named_modules():
+            if not (isinstance(module, nn.Linear) or isinstance(module, QScaleLinear) or isinstance(module, QLinear)):
+                continue
+            if not any(target_key in module_name for target_key in target_modules_list):
+                continue
+            logger.info(f"Adding {module_name} to GaLore parameters")
+            lowrank_params.append(module.weight)
+
+        id_lowrank_params = [id(p) for p in lowrank_params]
+        # make parameters without "rank" to another group
+        regular_params = [p for p in model.parameters() if id(p) not in id_lowrank_params]
+        # then call low rank optimizer
+        param_groups = [
+            {"params": regular_params},
+            {
+                "params": lowrank_params,
+                "rank": args.rank,
+                "update_proj_gap": args.update_proj_gap,
+                "scale": args.galore_scale,
+                "proj_type": args.proj_type,
+                "quant": args.proj_quant,
+                "quant_n_bit": args.proj_bits,
+                "quant_group_size": args.proj_group_size,
+                "cos_threshold": args.cos_threshold,
+                "gamma_proj": args.gamma_proj,
+                "queue_size": args.queue_size,
+            },
+        ]
+    elif "apollo" in args.optimizer.lower():
+        # make parameters with "rank" to a single group, if param_name has "mlp" or "attn"
+        lowrank_params = []
+        target_modules_list = ["attn", "mlp"]
+        for module_name, module in model.named_modules():
+            if not (isinstance(module, nn.Linear) or isinstance(module, QScaleLinear) or isinstance(module, QLinear)):
+                continue
+            if not any(target_key in module_name for target_key in target_modules_list):
+                continue
+            logger.info(f"Adding {module_name} to APOLLO parameters")
+            lowrank_params.append(module.weight)
+
+        id_lowrank_params = [id(p) for p in lowrank_params]
+        # make parameters without "rank" to another group
+        regular_params = [p for p in model.parameters() if id(p) not in id_lowrank_params]
+        # then call low rank optimizer
+        param_groups = [
+            {"params": regular_params},
+            {
+                "params": lowrank_params,
+                "rank": args.rank,
+                "update_proj_gap": args.update_proj_gap,
+                "scale": args.apollo_scale,
+                "proj_type": args.proj_type,
+                "proj": args.proj,
+                "scale_type": args.scale_type,
+            },
+        ]
+    elif "conda" in args.optimizer.lower():
+        # make parameters with "rank" to a single group, if param_name has "mlp" or "attn"
+        lowrank_params = []
+        target_modules_list = ["attn", "mlp"]
+        for module_name, module in model.named_modules():
+            if not (isinstance(module, nn.Linear) or isinstance(module, QScaleLinear) or isinstance(module, QLinear)):
+                continue
+            if not any(target_key in module_name for target_key in target_modules_list):
+                continue
+            logger.info(f"Adding {module_name} to conda parameters")
+            lowrank_params.append(module.weight)
+
+        id_lowrank_params = [id(p) for p in lowrank_params]
+        # make parameters without "rank" to another group
+        regular_params = [p for p in model.parameters() if id(p) not in id_lowrank_params]
+        # then call low rank optimizer
+        param_groups = [
+            {"params": regular_params},
+            {
+                "params": lowrank_params,
+                "rank": args.rank,
+                "update_proj_gap": args.update_proj_gap,
+                "scale": args.apollo_scale,
+                "proj_type": args.proj_type,
+                "proj": args.proj,
+                "scale_type": args.scale_type,
+            },
+        ]
+    else:
+        param_groups = None
+        id_lowrank_params = None
+
+    # print params and trainable params
+    logger.info(f"\n{model}\n")
+    logger.info(f"Total params: {sum(p.numel() for p in model.parameters()) / 1_000_000:.2f}M")
+
+    if args.simulation:
+        num_train_params = sum(p.numel() for p in trainable_params)
+    else:
+        num_train_params = sum(p.numel() for p in trainable_params) + sum(p.numel() for p in trainable_params_int8)
+
+    logger.info(f"Trainable params: {num_train_params / 1_000_000:.2f}M")
+    if "q_galore" in args.optimizer.lower():
+        logger.info(
+            f"Trainable params with Q-GaLore enabled: {sum(p.numel() for p in trainable_params_int8) / 1_000_000:.2f}M"
+        )
+    elif "galore" in args.optimizer.lower():
+        logger.info(f"Total params with GaLore enabled: {sum(p.numel() for p in lowrank_params) / 1_000_000:.2f}M")
+    elif "q_apollo" in args.optimizer.lower():
+        logger.info(
+            f"Trainable params with Q-APOLLO enabled: {sum(p.numel() for p in trainable_params_int8) / 1_000_000:.2f}M"
+        )
+    elif "apollo" in args.optimizer.lower():
+        logger.info(f"Total params with APOLLO enabled: {sum(p.numel() for p in lowrank_params) / 1_000_000:.2f}M")
+
+    logger.info(f"Saving model to {args.save_dir} every {args.save_every} update steps")
+
+    model, optimizer, scheduler, layer_wise_flag = setup_optimization(
+        args, model, trainable_params, param_groups, id_lowrank_params, model_config
+    )
+
+    if layer_wise_flag:
+        # will pass optimizer_dict and scheduler_dict out instead of optimizer and scheduler
+        optimizer_dict = optimizer
+        scheduler_dict = scheduler
+
+    # Bug-3 fix: wrap with DDP *before* torch.compile per PyTorch recommendation.
+    # This ensures gradient reduction hooks are correctly installed on the DDP module,
+    # and the compiled graph captures the full DDP+model forward pass.
+    # (Issue-5: optimizer.load_state_dict is called after both DDP and compile below.)
+    if not args.single_gpu:
+        model: LlamaForCausalLM = torch.nn.parallel.DistributedDataParallel(
+            model,
+            device_ids=[local_rank],
+            output_device=local_rank,
+            broadcast_buffers=False,
+        )
+
+    # compile the model (after DDP so the compiled graph includes DDP reduction)
+    if args.compile:
+        print("Compiling the model... (takes a ~minute)")
+        unoptimized_model = model
+
+        # Configure TorchDynamo to suppress errors and fall back to eager mode
+        import torch._dynamo
+        torch._dynamo.config.suppress_errors = args.dynamo_suppress_errors
+        torch._dynamo.config.verbose = False
+        # Set cache size limit to prevent memory issues during long training
+        torch._dynamo.config.cache_size_limit = args.dynamo_cache_limit
+
+        model = torch.compile(model) # requires PyTorch 2.0
+
+    # resume optimizer
+    if args.restore_optimizer and args.continue_from is not None:
+        logger.info("Restoring optimizer and scheduler from the checkpoint")
+        _optimizer_dir = args.continue_from
+        optimizer_checkpoint = torch.load(os.path.join(_optimizer_dir, "optimizer.pt"), map_location="cpu")
+        optimizer.load_state_dict(optimizer_checkpoint["optimizer"])
+        scheduler.load_state_dict(optimizer_checkpoint["scheduler"])
+        update_step = optimizer_checkpoint["update_step"]
+        beginning_step = update_step
+        global_step = optimizer_checkpoint["global_step"]
+        logger.info(f"Optimizer and scheduler restored from {_optimizer_dir}")
+
+    # ##############################
+    # TRAINING LOOP
+    # we use iterable dataset, so we may never go through all the data
+    # ##############################
+    # global steps and others are defined above
+    pad_idx = tokenizer.pad_token_id
+    update_time = time.time()
+    local_step = 0  # when continue_from is used, local_step != global_step
+    total_svd_count = 0
+
+    dataloader_iter = iter(dataloader)
+
+    # Issue-4 fix: accumulate loss across micro-batches so logged loss is the true
+    # gradient-accumulation average, not just the last micro-batch.
+    accumulated_loss = 0.0
+
+    # Skip data if resuming from checkpoint
+    if update_step != 0:
+        skip_batches = args.gradient_accumulation * update_step
+        logger.info(f"Skipping {skip_batches} batches to resume from update step {update_step}")
+        skipped = 0
+        for _ in range(skip_batches):
+            # Issue-6 fix: handle StopIteration during skip so all ranks stay aligned
+            try:
+                next(dataloader_iter)
+            except StopIteration:
+                logger.warning(
+                    f"Dataset exhausted during skip at batch {skipped}/{skip_batches}; "
+                    f"restarting iterator to keep ranks aligned."
+                )
+                dataloader_iter = iter(dataloader)
+                next(dataloader_iter)
+            skipped += 1
+        logger.info(f"Skipped {skipped} batches successfully")
+
+    while update_step <= args.num_training_steps:
+        try:
+            batch = next(dataloader_iter)
+        except StopIteration:
+            logger.info(f"Dataset completed one epoch. Starting new epoch with reshuffled data.")
+            dataloader_iter = iter(dataloader)
+            batch = next(dataloader_iter)
+
+        global_step += 1
+        local_step += 1
+
+        if update_step >= args.num_training_steps:
+            logger.info(f"Reached max number of update steps ({args.num_training_steps}). Stopping training.")
+            logger.info(f"Rank {global_rank} stopping training.")
+            break
+
+        # forward & backward
+        batch = {k: v.to(device) for k, v in batch.items()}
+        labels = batch["input_ids"].clone()
+        labels[labels == pad_idx] = -100
+        tokens_seen += (batch["input_ids"] != pad_idx).sum().item() * world_size
+
+        loss = model(**batch, labels=labels).loss
+
+        scaled_loss = loss / args.gradient_accumulation
+        scaled_loss.backward()
+        accumulated_loss += loss.item()  # Issue-4: accumulate before the continue
+
+        if global_step % args.gradient_accumulation != 0:
+            continue
+
+        # The below code is only executed during the update step
+        # Issue-4: compute average loss over all micro-batches in this accumulation window
+        avg_loss = accumulated_loss / args.gradient_accumulation
+        accumulated_loss = 0.0  # reset for next accumulation window
+        # add grad clipping: TODO: add gradient clipping of int8 weight
+        if args.grad_clipping != 0.0:
+            torch.nn.utils.clip_grad_norm_(trainable_params, args.grad_clipping)
+        # Periodic memory cleanup to prevent symbolic tensor issues during long training
+        if global_step % args.memory_cleanup_frequency == 0:
+            torch.cuda.empty_cache()
+            # Clear TorchDynamo cache to prevent memory accumulation
+            if args.compile:
+                import torch._dynamo
+                torch._dynamo.reset()
+
+        if global_rank == 0:
+            pbar.update(1)
+        if not layer_wise_flag:  # layer-wise updation is done during backward; requires gradient_accumulation equals 1
+            optimizer.step()
+            scheduler.step()
+            optimizer.zero_grad()
+
+        update_step += 1
+        update_time = time.time() - update_time
+
+        # save checkpoint by save_every
+        if local_step > args.gradient_accumulation and update_step % args.save_every == 0 and global_rank == 0:
+            current_model_directory = f"{args.save_dir}/model_{update_step}"
+            logger.info(f"Saving model and optimizer to {current_model_directory}, update step {update_step}")
+            os.makedirs(args.save_dir, exist_ok=True)
+            # Bug-1 fix: unwrap DDP/compiled model for saving; works in both single-GPU and multi-GPU modes
+            unwrapped_model = model.module if hasattr(model, 'module') else model
+            unwrapped_model.save_pretrained(current_model_directory, max_shard_size="500GB", from_pt=True)
+            saving_model_weight(unwrapped_model, f"{current_model_directory}/pytorch_model.bin", args)
+
+            optimizer_checkpoint = {
+                "optimizer": optimizer.state_dict(),
+                "scheduler": scheduler.state_dict(),
+                "update_step": update_step,
+                "global_step": global_step,
+                "config": run_config,
+                "wandb": wandb.run.dir if not args.unset_wandb else None,
+                "dtype": args.dtype,
+            }
+            torch.save(optimizer_checkpoint, f"{current_model_directory}/optimizer.pt")
+
+            training_state_checkpoint = {
+                "global_step": global_step,
+                "update_step": update_step,
+                "tokens_seen": tokens_seen,
+                "tokens_seen_before": tokens_seen_before,
+                "update_time": update_time,
+            }
+            with open(f"{current_model_directory}/training_state.json", "w") as f:
+                json.dump(training_state_checkpoint, f, indent=4)
+
+            # save wandb related info
+            if not args.unset_wandb:
+                wandb_info = {
+                    "wandb_id": wandb.run.id,
+                }
+                with open(f"{args.save_dir}/wandb.json", "w") as f:
+                    json.dump(wandb_info, f, indent=4)
+
+        # evaluation
+        if update_step % args.eval_every == 0:
+            logger.info(f"Performing evaluation at step {update_step}")
+            total_loss, evaluated_on_tokens, perplexity = evaluate_model(
+                model, tokenizer, pad_idx, global_rank, world_size, device, args
+            )
+
+            if global_rank == 0:
+                if not args.unset_wandb:
+                    wandb.log(
+                        {
+                            "eval_loss": total_loss,
+                            "eval_perplexity": perplexity,
+                            "eval_tokens": evaluated_on_tokens,
+                        },
+                        step=update_step,
+                    )
+            logger.info(f"Eval loss at step {update_step}: {total_loss}, Eval perplexity: {perplexity}")
+
+        if not layer_wise_flag:
+            lr = optimizer.param_groups[0]["lr"]
+        else:
+            lr = list(optimizer_dict.values())[0].param_groups[0]["lr"]
+        tokens_in_update = tokens_seen - tokens_seen_before
+        tokens_seen_before = tokens_seen
+        batches_in_update = args.gradient_accumulation * world_size
+        if not layer_wise_flag:
+            total_svd_count = getting_svd_cnt(optimizer)
+        else:
+            total_svd_count = 0
+
+        if global_rank == 0:
+            if not args.unset_wandb:
+                wandb.log(
+                    {
+                        "loss": avg_loss,
+                        "lr": lr,
+                        "update_step": update_step,
+                        "tokens_seen": tokens_seen,
+                        "total_svd_count": total_svd_count,
+                        "throughput_tokens": tokens_in_update / update_time,
+                        "throughput_examples": args.total_batch_size / update_time,
+                        "throughput_batches": batches_in_update / update_time,
+                    },
+                    step=update_step,
+                )
+        update_time = time.time()
+
+    # ##############################
+    # END of training loop
+    # ##############################
+    logger.info("Training finished")
+    if global_rank == 0:
+        pbar.close()
+
+    current_model_directory = f"{args.save_dir}/model_{update_step}"
+    if global_rank == 0 and not os.path.exists(current_model_directory):
+        logger.info(f"Saving model and optimizer to {current_model_directory}, update step {update_step}")
+        os.makedirs(args.save_dir, exist_ok=True)
+        # Bug-1 fix: unwrap DDP/compiled model for saving; works in both single-GPU and multi-GPU modes
+        unwrapped_model = model.module if hasattr(model, 'module') else model
+        unwrapped_model.save_pretrained(current_model_directory, max_shard_size="500GB", from_pt=True)
+        saving_model_weight(unwrapped_model, f"{current_model_directory}/pytorch_model.bin", args)
+
+        optimizer_checkpoint = {
+            "optimizer": optimizer.state_dict(),
+            "scheduler": scheduler.state_dict(),
+            "update_step": update_step,
+            "global_step": global_step,
+            "config": run_config,
+            "wandb": wandb.run.dir if not args.unset_wandb else None,
+            "dtype": args.dtype,
+        }
+        torch.save(optimizer_checkpoint, f"{current_model_directory}/optimizer.pt")
+
+        training_state_checkpoint = {
+            "global_step": global_step,
+            "update_step": update_step,
+            "tokens_seen": tokens_seen,
+            "tokens_seen_before": tokens_seen_before,
+            "update_time": update_time,
+        }
+        with open(f"{current_model_directory}/training_state.json", "w") as f:
+            json.dump(training_state_checkpoint, f, indent=4)
+
+    # Final evaluation
+    logger.info("Running final evaluation")
+    model.eval()
+    del loss, optimizer, scheduler
+    import gc
+
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    total_loss, evaluated_on_tokens, perplexity = evaluate_model(model, tokenizer, pad_idx, global_rank, world_size, device, args)
+
+    if global_rank == 0:
+        if not args.unset_wandb:
+            wandb.log(
+                {
+                    "final_eval_loss": total_loss,
+                    "final_eval_perplexity": perplexity,
+                    "final_eval_tokens": evaluated_on_tokens,
+                },
+                step=update_step,
+            )
+        logger.info(f"Final eval loss: {total_loss}, Final eval perplexity: {perplexity}")
+
+    logger.info("Script finished successfully")
+    print(f"Rank {global_rank} finished successfully")
+
+
+if __name__ == "__main__":
+    print("Starting script")
+    args = parse_args(None)
+    main(args)

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/files/requirements.txt

@@ -0,0 +1,134 @@
+aria2==0.0.1b0
+anyio==4.12.0
+setuptools==80.9.0
+torchvision==0.20.1+cu121
+pyarrow==20.0.0
+peft==0.17.1
+conda-pack==0.8.1
+scikit-learn==1.6.1
+pyparsing==3.3.1
+sympy==1.13.1
+typer-slim==0.20.1
+pip==25.1.1
+pip==25.3
+fonttools==4.60.2
+packaging==25.0
+click==8.1.8
+accelerate==1.10.1
+psutil==7.2.0
+wheel==0.45.1
+multidict==6.7.0
+requests==2.32.5
+async-timeout==5.0.1
+triton==3.1.0
+loguru==0.7.3
+aiohappyeyeballs==2.6.1
+sentry-sdk==2.48.0
+annotated-types==0.7.0
+certifi==2025.11.12
+nvidia-curand-cu12==10.3.2.106
+shellingham==1.5.4
+package_name==0.1
+wandb==0.23.0
+nvitop==1.6.1
+nvidia-nccl-cu12==2.21.5
+nvidia-cublas-cu12==12.1.3.1
+tokenizers==0.22.1
+nvidia-cusparse-cu12==12.1.0.106
+scipy==1.13.1
+propcache==0.4.1
+nvidia-ml-py==13.580.82
+typing_extensions==4.15.0
+sac==0.1.0
+torch-optimizer==0.3.0
+aria2==0.0.1b0
+h11==0.16.0
+pillow==11.3.0
+PyYAML==6.0.3
+six==1.17.0
+GitPython==3.1.45
+addict==2.4.0
+seaborn==0.13.2
+filelock==3.19.1
+modelscope==1.33.0
+et_xmlfile==2.0.0
+regex==2025.11.3
+nvidia-cufft-cu12==11.0.2.54
+nvidia-cuda-cupti-cu12==12.1.105
+lion-pytorch==0.2.3
+matplotlib==3.9.4
+pandas==2.3.2
+gitdb==4.0.12
+kiwisolver==1.4.7
+idna==3.11
+numpy==2.0.2
+nvidia-cuda-runtime-cu12==12.1.105
+httpx==0.28.1
+frozenlist==1.8.0
+smmap==5.0.2
+datasets==2.14.0
+yarl==1.22.0
+eval_type_backport==0.3.1
+nvidia-cuda-nvrtc-cu12==12.1.105
+huggingface-hub==0.36.0
+torchaudio==2.5.1+cu121
+aiosignal==1.4.0
+importlib_resources==6.5.2
+nvidia-cusolver-cu12==11.4.5.107
+networkx==3.2.1
+tzdata==2025.3
+bitsandbytes==0.42.0
+cycler==0.12.1
+jq==1.10.0
+mpmath==1.3.0
+pydantic_core==2.41.5
+nvidia-cudnn-cu12==9.1.0.70
+typing-inspection==0.4.2
+httpcore==1.0.9
+nvidia-nvtx-cu12==12.1.105
+platformdirs==4.4.0
+MarkupSafe==2.1.5
+multiprocess==0.70.15
+zipp==3.23.0
+transformers==4.57.3
+nvidia-nvjitlink-cu12==12.9.86
+exceptiongroup==1.3.1
+pydantic==2.12.5
+charset-normalizer==3.4.4
+joblib==1.5.3
+dill==0.3.7
+fsspec==2023.9.2
+torch==2.5.1+cu121
+aiohttp==3.13.2
+urllib3==2.6.2
+apollo-torch==1.0.3
+contourpy==1.3.0
+evaluate==0.4.6
+attrs==25.4.0
+pytz==2025.2
+safetensors==0.7.0
+pytorch-ranger==0.1.1
+threadpoolctl==3.6.0
+Jinja2==3.1.6
+protobuf==6.33.2
+python-dateutil==2.9.0.post0
+xxhash==3.6.0
+openpyxl==3.1.5
+hf-xet==1.2.0
+tqdm==4.67.1
+jaraco.context==5.3.0
+platformdirs==4.2.2
+importlib_metadata==8.0.0
+more-itertools==10.3.0
+typing_extensions==4.12.2
+autocommand==2.2.2
+wheel==0.45.1
+zipp==3.19.2
+packaging==24.2
+backports.tarfile==1.2.0
+inflect==7.3.1
+typeguard==4.3.0
+jaraco.functools==4.0.1
+jaraco.collections==5.1.0
+jaraco.text==3.12.1
+tomli==2.0.1

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/logs/debug-core.log

@@ -0,0 +1,196 @@
+{"time":"2026-04-19T23:47:18.021764833+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpo3l032g6/port-98537.txt","pid":98537,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.029666993+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":98537}
+{"time":"2026-04-19T23:47:18.029927965+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-98537-99286-355652662/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.048284469+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.078185493+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpnvqgmy5i/port-98542.txt","pid":98542,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.078992412+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":98542}
+{"time":"2026-04-19T23:47:18.078970839+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-98542-99304-3007006831/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.080307948+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"21tlsf0i","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.135684998+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprq6n2ybz/port-76073.txt","pid":76073,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.137529876+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":76073}
+{"time":"2026-04-19T23:47:18.137591396+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-76073-76853-466182552/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.187427529+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.220860482+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.209949557+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmprok4hglu/port-99742.txt","pid":99742,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.209951784+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpjiasccxf/port-99748.txt","pid":99748,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.211453276+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":99742}
+{"time":"2026-04-19T23:47:18.211457149+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":99748}
+{"time":"2026-04-19T23:47:18.211514611+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-99742-101551-3085090515/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.218061611+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"yiy34l67","id":"1(@)"}
+-101562-3832278309/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.269877528+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"fdnbp7cl","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.271090519+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.283121843+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"21tlsf0i","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.286926828+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpggxenr18/port-111040.txt","pid":111040,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.28693122+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpsnt1dchv/port-111045.txt","pid":111045,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.300789277+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpc3ed2r3u/port-111854.txt","pid":111854,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.302607546+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":111854}
+{"time":"2026-04-19T23:47:18.302683903+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-111854-112701-4238633932/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.305913132+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpmfcwim_m/port-76068.txt","pid":76068,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.306960611+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":76068}
+{"time":"2026-04-19T23:47:18.306953648+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-76068-76876-1915591522/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.306095082+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"85n8tjn3","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.314845902+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.288617729+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":111040}
+{"time":"2026-04-19T23:47:18.327282252+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":111045}
+{"time":"2026-04-19T23:47:18.288631687+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-111040-112000-1180210445/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.327291275+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-111045-112001-2453585409/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.346654253+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"vrvs25ab","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.381947133+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.39046355+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.397039905+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.401615357+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"yiy34l67","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.410483286+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.415315815+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"22i9c0l8","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.421928169+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"hjmiiwpe","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.42686866+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"dhgtud9k","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.439541955+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphtl9f_ni/port-111849.txt","pid":111849,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.440374904+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":111849}
+{"time":"2026-04-19T23:47:18.4403598+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-111849-112720-568316116/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.44697126+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"epujs7qw","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.461429221+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"fdnbp7cl","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.526899886+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.559484533+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"pafttcq9","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.581911568+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpo4jua6ux/port-98798.txt","pid":98798,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.581921661+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp9zkkecpk/port-98792.txt","pid":98792,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.583091474+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":98798}
+{"time":"2026-04-19T23:47:18.583314014+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":98792}
+{"time":"2026-04-19T23:47:18.583106123+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-98798-99747-3528434782/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.583312585+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-98792-99748-2045978848/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.617541968+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"dhgtud9k","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.618998938+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"22i9c0l8","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.636792403+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"hjmiiwpe","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.641453678+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"epujs7qw","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.675611891+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.680131324+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpiw5twogx/port-88639.txt","pid":88639,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.681711769+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":88639}
+{"time":"2026-04-19T23:47:18.681746341+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-88639-89639-2998582682/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.685958921+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.710526608+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"yljaz7hy","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.704563215+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"85n8tjn3","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.719501488+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"dujhio3b","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.732251608+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"vrvs25ab","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.751633063+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"pafttcq9","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.75616279+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.797761417+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"xqvrxii2","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.834384637+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmph_rkjoy0/port-88645.txt","pid":88645,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-19T23:47:18.835396377+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":88645}
+{"time":"2026-04-19T23:47:18.835405+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-88645-89660-346707904/socket","Net":"unix"}}
+{"time":"2026-04-19T23:47:18.927501062+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"yljaz7hy","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.930167495+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"dujhio3b","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.933169858+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-19T23:47:18.969120085+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"r75330cz","id":"1(@)"}
+{"time":"2026-04-19T23:47:19.023847579+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"xqvrxii2","id":"1(@)"}
+{"time":"2026-04-19T23:47:19.172783549+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"r75330cz","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.146604788+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.147864857+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.148205319+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T01:08:49.148618805+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.149735433+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-98537-99286-355652662/socket","Net":"unix"}}
+{"time":"2026-04-20T01:08:49.154374103+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.154860689+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.155311814+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T01:08:49.253458485+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.25398645+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T01:08:49.253975474+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.254456653+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-99742-101551-3085090515/socket","Net":"unix"}}
+{"time":"2026-04-20T01:08:49.254838526+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.314298109+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.314745275+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T01:08:49.31510503+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T01:09:04.323031847+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T01:09:04.323740325+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T01:09:04.324357481+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T01:09:04.323760517+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T01:09:04.325195807+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-99748-101562-3832278309/socket","Net":"unix"}}
+{"time":"2026-04-20T01:09:04.326737777+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T01:09:04.327166361+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T01:09:04.327563129+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T01:09:11.41740719+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T01:09:11.417884577+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T01:09:11.418297371+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T01:09:11.417905717+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T01:09:11.419118842+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-98542-99304-3007006831/socket","Net":"unix"}}
+{"time":"2026-04-20T01:09:11.421316849+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T01:09:11.421698457+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T01:09:11.42207015+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T01:10:48.070345078+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T01:10:48.070932583+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T01:10:48.071464299+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T01:10:48.070946329+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T01:10:48.072287025+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-111849-112720-568316116/socket","Net":"unix"}}
+{"time":"2026-04-20T01:10:48.074076319+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T01:10:48.074486591+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T01:10:48.074881194+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T01:10:55.161432161+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T01:10:55.162122358+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T01:10:55.162108411+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T01:10:55.162692909+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-111854-112701-4238633932/socket","Net":"unix"}}
+{"time":"2026-04-20T01:10:55.163102651+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T01:10:55.166362966+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T01:10:55.166838401+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T01:10:55.167285135+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T01:11:27.283435067+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T01:11:27.284057323+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T01:11:27.28403544+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T01:11:27.284585607+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-98792-99748-2045978848/socket","Net":"unix"}}
+{"time":"2026-04-20T01:11:27.28520036+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T01:11:27.287750736+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T01:11:27.288560815+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T01:11:27.288970908+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T01:11:42.295715754+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T01:11:42.296280122+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T01:11:42.296821181+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T01:11:42.296291028+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T01:11:42.297786054+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-98798-99747-3528434782/socket","Net":"unix"}}
+{"time":"2026-04-20T01:11:42.300652964+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T01:11:42.301119878+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T01:11:42.301481215+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T09:55:25.042583189+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T09:55:25.043601928+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T09:55:25.043597224+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T09:55:25.044129205+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-111040-112000-1180210445/socket","Net":"unix"}}
+{"time":"2026-04-20T09:55:25.04456338+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T09:55:25.048450045+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T09:55:25.048937508+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T09:55:25.049364415+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T09:55:26.509170259+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T09:55:26.509792575+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T09:55:26.509787671+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T09:55:26.510719316+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T09:55:26.510282741+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-111045-112001-2453585409/socket","Net":"unix"}}
+{"time":"2026-04-20T09:55:26.513872593+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T09:55:26.514283106+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T09:55:26.514665103+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T09:57:18.487273701+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T09:57:18.48779167+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T09:57:18.487782288+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T09:57:18.488283466+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-76073-76853-466182552/socket","Net":"unix"}}
+{"time":"2026-04-20T09:57:18.488647164+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T09:57:18.500524213+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T09:57:18.500969049+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T09:57:18.501359083+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T09:57:34.947234839+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T09:57:34.947735354+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T09:57:34.947723917+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T09:57:34.94818644+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-76068-76876-1915591522/socket","Net":"unix"}}
+{"time":"2026-04-20T09:57:34.94854198+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T09:57:34.950855281+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T09:57:34.951263279+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T09:57:34.951650899+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T10:01:41.014666105+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T10:01:41.0152138+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T10:01:41.015205054+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T10:01:41.01569071+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-88639-89639-2998582682/socket","Net":"unix"}}
+{"time":"2026-04-20T10:01:41.01608462+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T10:01:41.019915307+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T10:01:41.020254481+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T10:01:41.02057197+08:00","level":"INFO","msg":"server is closed"}
+{"time":"2026-04-20T10:02:09.247682946+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-20T10:02:09.248225032+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-20T10:02:09.248235825+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-20T10:02:09.248683569+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-20T10:02:09.249161406+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-88645-89660-346707904/socket","Net":"unix"}}
+{"time":"2026-04-20T10:02:09.251438194+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-20T10:02:09.251829309+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-20T10:02:09.252180149+08:00","level":"INFO","msg":"server is closed"}

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/logs/debug-internal.log

@@ -0,0 +1,12 @@
+{"time":"2026-04-19T23:47:18.48078063+08:00","level":"INFO","msg":"stream: starting","core version":"0.23.0"}
+{"time":"2026-04-19T23:47:18.612202918+08:00","level":"WARN","msg":"featurechecker: GraphQL client is nil, skipping feature loading"}
+{"time":"2026-04-19T23:47:18.612265953+08:00","level":"INFO","msg":"stream: created new stream","id":"dhgtud9k"}
+{"time":"2026-04-19T23:47:18.612302459+08:00","level":"INFO","msg":"handler: started","stream_id":"dhgtud9k"}
+{"time":"2026-04-19T23:47:18.617535039+08:00","level":"INFO","msg":"stream: started","id":"dhgtud9k"}
+{"time":"2026-04-19T23:47:18.617543876+08:00","level":"INFO","msg":"writer: started","stream_id":"dhgtud9k"}
+{"time":"2026-04-19T23:47:18.617549943+08:00","level":"INFO","msg":"sender: started","stream_id":"dhgtud9k"}
+{"time":"2026-04-19T23:47:18.618872114+08:00","level":"WARN","msg":"runupserter: server does not expand metric globs but the x_server_side_expand_glob_metrics setting is set; ignoring"}
+{"time":"2026-04-20T09:55:26.509801244+08:00","level":"INFO","msg":"stream: closing","id":"dhgtud9k"}
+{"time":"2026-04-20T09:55:26.510476014+08:00","level":"INFO","msg":"handler: closed","stream_id":"dhgtud9k"}
+{"time":"2026-04-20T09:55:26.511606678+08:00","level":"INFO","msg":"sender: closed","stream_id":"dhgtud9k"}
+{"time":"2026-04-20T09:55:26.511618561+08:00","level":"INFO","msg":"stream: closed","id":"dhgtud9k"}

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/logs/debug.log

@@ -0,0 +1,24 @@
+2026-04-19 23:47:17,983 INFO    MainThread:111045 [wandb_setup.py:_flush():80] Current SDK version is 0.23.0
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_setup.py:_flush():80] Configure stats pid to 111045
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_setup.py:_flush():80] Loading settings from /mnt/petrelfs/panjiabao/.config/wandb/settings
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_setup.py:_flush():80] Loading settings from /mnt/petrelfs/panjiabao/Optimizer/SAC/wandb/settings
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_setup.py:_flush():80] Loading settings from environment variables
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_init.py:setup_run_log_directory():713] Logging user logs to /mnt/dhwfile/tancheng/panjiabao/Result/SAC_C4/work_dirs/350m/apollo_sweep_lr1e-2_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/logs/debug.log
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_init.py:setup_run_log_directory():714] Logging internal logs to /mnt/dhwfile/tancheng/panjiabao/Result/SAC_C4/work_dirs/350m/apollo_sweep_lr1e-2_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/logs/debug-internal.log
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_init.py:init():840] calling init triggers
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_init.py:init():845] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-04-19 23:47:17,984 INFO    MainThread:111045 [wandb_init.py:init():888] starting backend
+2026-04-19 23:47:18,397 INFO    MainThread:111045 [wandb_init.py:init():891] sending inform_init request
+2026-04-19 23:47:18,425 INFO    MainThread:111045 [wandb_init.py:init():899] backend started and connected
+2026-04-19 23:47:18,429 INFO    MainThread:111045 [wandb_init.py:init():969] updated telemetry
+2026-04-19 23:47:18,474 INFO    MainThread:111045 [wandb_init.py:init():993] communicating run to backend with 90.0 second timeout
+2026-04-19 23:47:18,620 INFO    MainThread:111045 [wandb_init.py:init():1040] starting run threads in backend
+2026-04-19 23:47:19,015 INFO    MainThread:111045 [wandb_run.py:_console_start():2504] atexit reg
+2026-04-19 23:47:19,015 INFO    MainThread:111045 [wandb_run.py:_redirect():2352] redirect: wrap_raw
+2026-04-19 23:47:19,015 INFO    MainThread:111045 [wandb_run.py:_redirect():2421] Wrapping output streams.
+2026-04-19 23:47:19,015 INFO    MainThread:111045 [wandb_run.py:_redirect():2444] Redirects installed.
+2026-04-19 23:47:19,023 INFO    MainThread:111045 [wandb_init.py:init():1080] run started, returning control to user process
+2026-04-19 23:47:27,035 INFO    MainThread:111045 [wandb_run.py:_config_callback():1385] config_cb None None {'model_config': 'configs/llama_350m.json', 'exp_config': 'exp_v2/configs/llama_350m_apollo.json', 'eval_every': 1000, 'save_every': 60000, 'dtype': 'bfloat16', 'seed': 0, 'compile': True, 'dynamo_suppress_errors': True, 'dynamo_cache_limit': 10000, 'memory_cleanup_frequency': 10000, 'resume_step': None, 'restore_optimizer': False, 'continue_from': None, 'single_gpu': False, 'save_dir': '/mnt/dhwfile/tancheng/panjiabao/Result/SAC_C4/work_dirs/350m/apollo_sweep_lr1e-2_20260419_234621', 'use_hf_model': False, 'workers': 12, 'batch_size': 128, 'gradient_accumulation': 1, 'total_batch_size': 512, 'warmup_steps': 6000, 'num_training_steps': 60000, 'max_train_tokens': None, 'optimizer': 'apollo_adamw', 'max_length': 256, 'scheduler': 'cosine', 'min_lr_ratio': 0.1, 'weight_decay': 0.0, 'grad_clipping': 0.0, 'activation_checkpointing': False, 'data_path': '/mnt/dhwfile/tancheng/panjiabao/dataset/C4/en', 'data_name': 'en', 'tags': None, 'name': 'test', 'project': 'test', 'unset_wandb': False, 'entity': None, 'wandb_dir': '/mnt/dhwfile/tancheng/panjiabao/Result/SAC_C4/work_dirs/350m/apollo_sweep_lr1e-2_20260419_234621', 'beta1': 0.9, 'beta2': 0.99, 'beta3': 0.99, 'eps': 1e-06, 'rank': 256, 'update_proj_gap': 200, 'galore_scale': 1.0, 'proj_type': 'std', 'proj_quant': False, 'proj_bits': 8, 'proj_group_size': 256, 'weight_quant': False, 'weight_bits': 8, 'weight_group_size': 256, 'stochastic_round': False, 'simulation': False, 'cos_threshold': 1, 'gamma_proj': 2, 'queue_size': 5, 'proj': 'random', 'scale_type': 'channel', 'apollo_scale': 1, 'scale_front': False, 'n_clusters': 3, 'scale_update_freq': 500, 'scale_level': '1,0,1,1', 'scale_bound': None, 'metric': 'mean', 'align_grad': False, 'dim': 4096, 'n_heads': 32, 'muon_ns_steps': 5, 'muon_momentum': 0.95, 'nproc_per_node': 4, 'max_lr': 0.01, 'total_params_M': 367.96928, 'dataset': 'c4', 'model': {'vocab_size': 32000, 'max_position_embeddings': 2048, 'hidden_size': 1024, 'intermediate_size': 2736, 'num_hidden_layers': 24, 'num_attention_heads': 16, 'num_key_value_heads': 16, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 10000.0, 'rope_scaling': None, 'attention_bias': False, 'attention_dropout': 0.0, 'mlp_bias': False, 'head_dim': 64, 'return_dict': True, 'output_hidden_states': False, 'torchscript': False, 'dtype': None, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'architectures': ['LLaMAForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'task_specific_params': None, 'problem_type': None, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 0, 'pad_token_id': -1, 'eos_token_id': 1, 'sep_token_id': None, 'decoder_start_token_id': None, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'num_beam_groups': 1, 'diversity_penalty': 0.0, '_name_or_path': 'configs/llama_350m.json', 'transformers_version': '4.57.3', 'max_sequence_length': 1024, 'model_type': 'llama', 'tf_legacy_loss': False, 'use_bfloat16': False, 'output_attentions': False}, 'world_size': 4, 'device': 'cuda:0'}
+2026-04-20 09:55:26,509 INFO    wandb-AsyncioManager-main:111045 [service_client.py:_forward_responses():80] Reached EOF.
+2026-04-20 09:55:26,510 INFO    wandb-AsyncioManager-main:111045 [mailbox.py:close():137] Closing mailbox, abandoning 0 handles.

350m/apollo_lr1e_2_b1_0_9_b2_0_99_eps_1e_6_scale_1_rank_256_T_200_A100_ppl_16_4294_20260419_234621/wandb/offline-run-20260419_234717-dhgtud9k/run-dhgtud9k.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28ea4d8c8e24a0220fddb167b682cecf4d1e93c2cd9fab1a020e785a641a256c
+size 63354972