Add files using upload-large-folder tool

routing_rerun/N_2.5/.hydra/config.yaml

@@ -0,0 +1,57 @@
+model:
+  config_path: ${oc.env:PROJECT_ROOT}/hnet_project/configs/hnet_2stage_XL_code.json
+  checkpoint_path: ${oc.env:PROJECT_ROOT}/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+training:
+  epochs: 3
+  max_steps: null
+  batch_size: 8
+  eval_batch_size: 24
+  gradient_accumulation_steps: 4
+  lr: 0.0001
+  weight_decay: 0.1
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-08
+  lr_scheduler: wsd
+  warmup_ratio: 0.1
+  decay_ratio: 0.2
+  warmup_steps: 100
+  min_lr_ratio: 0.1
+  lr_multiplier:
+  - 2.0
+  - 1.5
+  - 1.0
+  load_balancing_weight: 0.5
+  load_balancing_N: 2.5
+  max_grad_norm: 1.0
+  use_amp: true
+  resume: false
+  resume_checkpoint: null
+  warmup_model: true
+data:
+  path: ${oc.env:PROJECT_ROOT}/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 0
+  pin_memory: true
+  max_train_samples: 150000
+  max_val_samples: null
+logging:
+  log_interval: 10
+  save_interval: 3000
+  eval_interval: 1000
+  save_every_epoch: false
+  model_only_checkpoints: true
+tracking:
+  enabled: true
+  backend: wandb
+  project: routing-evolution
+  run_name: routing_N2.5
+  entity: null
+  base_url: https://wandb.platun0v.ru
+  local_dir: ${paths.output_dir}
+paths:
+  output_dir: outputs/${now:%Y-%m-%d}/N_${training.load_balancing_N}
+seed: 42
+device: cuda

routing_rerun/N_2.5/.hydra/hydra.yaml

@@ -0,0 +1,162 @@
+hydra:
+  run:
+    dir: ${paths.output_dir}
+  sweep:
+    dir: outputs/multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - training.load_balancing_N=2.5
+    - tracking.run_name=routing_N2.5
+  job:
+    name: train
+    chdir: false
+    override_dirname: tracking.run_name=routing_N2.5,training.load_balancing_N=2.5
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace/byte-llms-code/routing_evolution_exp
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/byte-llms-code/routing_evolution_exp/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/byte-llms-code/routing_evolution_exp/outputs/2026-05-23/N_2.5
+    choices:
+      paths: default
+      tracking: default
+      logging: default
+      data: default
+      training: default
+      model: hnet_xl_code
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

routing_rerun/N_2.5/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- training.load_balancing_N=2.5
+- tracking.run_name=routing_N2.5

routing_rerun/N_2.5/eval_results/eval_config.yaml

@@ -0,0 +1,29 @@
+model:
+  config_path: /workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json
+  checkpoint_path: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+data:
+  path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 0
+  pin_memory: true
+  max_train_samples: 150000
+  max_val_samples: null
+evaluation:
+  batch_size: 16
+  max_samples: 2000
+  compute_bpb: true
+  bleu_tokenize: none
+  use_amp: true
+  save_predictions: true
+generation:
+  max_length: 256
+  temperature: 0.1
+  top_k: 0
+  top_p: 1.0
+paths:
+  run_dir: outputs/2026-05-23/N_2.5
+  eval_initial: true
+  eval_final: true
+seed: 42
+device: cuda

routing_rerun/N_2.5/eval_results/metrics_model_best.txt

@@ -0,0 +1,17 @@
+Checkpoint: model_best (outputs/2026-05-23/N_2.5/model_best.pt)
+================================================================================
+
+exact_match: 0.3755
+token_accuracy: 0.4797733652504865
+bleu: 19.493683848607724
+bpb: 1.328336302401117
+num_samples: 2000
+gen_wall_time_s: 377.6293431438389
+gen_samples_per_s: 5.2961986040322095
+gen_time_per_sample_ms: 188.81467157191946
+gen_chars_per_s: 145.518352844389
+gen_batch_mean_ms: 3021.0347451507114
+gen_batch_p50_ms: 2677.273398032412
+gen_batch_p95_ms: 6516.014387598256
+gen_batch_max_ms: 14755.625324964058
+gen_num_batches: 125

routing_rerun/N_2.5/eval_results/metrics_model_final.txt

@@ -0,0 +1,17 @@
+Checkpoint: model_final (outputs/2026-05-23/N_2.5/model_final.pt)
+================================================================================
+
+exact_match: 0.3565
+token_accuracy: 0.475812423579658
+bleu: 18.406700285414495
+bpb: 1.4244987555298723
+num_samples: 2000
+gen_wall_time_s: 461.3162157420302
+gen_samples_per_s: 4.335420979691743
+gen_time_per_sample_ms: 230.6581078710151
+gen_chars_per_s: 122.02909431538349
+gen_batch_mean_ms: 3690.5297259362414
+gen_batch_p50_ms: 3152.6444799965248
+gen_batch_p95_ms: 7729.715668002599
+gen_batch_max_ms: 16093.165102996863
+gen_num_batches: 125

routing_rerun/N_2.5/eval_results/metrics_pretrained.txt

@@ -0,0 +1,17 @@
+Checkpoint: pretrained (/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt)
+================================================================================
+
+exact_match: 0.0
+token_accuracy: 0.46733945270119
+bleu: 1.1096613769578676
+bpb: 1.876943169957491
+num_samples: 2000
+gen_wall_time_s: 1815.3716653165757
+gen_samples_per_s: 1.1017027742642593
+gen_time_per_sample_ms: 907.6858326582878
+gen_chars_per_s: 281.1892516296283
+gen_batch_mean_ms: 14522.973322532605
+gen_batch_p50_ms: 14979.340799036436
+gen_batch_p95_ms: 15534.628321009222
+gen_batch_max_ms: 34090.05431999685
+gen_num_batches: 125

routing_rerun/N_2.5/eval_results/summary.txt

@@ -0,0 +1,8 @@
+EVALUATION SUMMARY
+=============================================================================
+
+Checkpoint            Exact Match    Token Acc     BLEU        BPB  ms/sample
+-----------------------------------------------------------------------------
+pretrained                  0.00%       46.73%     1.11     1.8769      907.7
+model_best                 37.55%       47.98%    19.49     1.3283      188.8
+model_final                35.65%       47.58%    18.41     1.4245      230.7

routing_rerun/N_2.5/wandb/debug-internal.log

@@ -0,0 +1,13 @@
+{"time":"2026-05-23T17:28:20.034273503Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-05-23T17:28:20.495979525Z","level":"INFO","msg":"stream: created new stream","id":"hcp9axm4"}
+{"time":"2026-05-23T17:28:20.496106936Z","level":"INFO","msg":"handler: started","stream_id":"hcp9axm4"}
+{"time":"2026-05-23T17:28:20.496281243Z","level":"INFO","msg":"stream: started","id":"hcp9axm4"}
+{"time":"2026-05-23T17:28:20.496287355Z","level":"INFO","msg":"writer: started","stream_id":"hcp9axm4"}
+{"time":"2026-05-23T17:28:20.496309559Z","level":"INFO","msg":"sender: started","stream_id":"hcp9axm4"}
+{"time":"2026-05-23T17:28:20.630740567Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-24T00:08:43.285793193Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-24T00:08:43.389899668Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-24T00:08:43.395126237Z","level":"INFO","msg":"stream: closing","id":"hcp9axm4"}
+{"time":"2026-05-24T00:08:43.395149698Z","level":"INFO","msg":"handler: closed","stream_id":"hcp9axm4"}
+{"time":"2026-05-24T00:08:43.395348295Z","level":"INFO","msg":"sender: closed","stream_id":"hcp9axm4"}
+{"time":"2026-05-24T00:08:43.395364596Z","level":"INFO","msg":"stream: closed","id":"hcp9axm4"}

routing_rerun/N_2.5/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-05-23 17:28:19,720 INFO    MainThread:15014 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_setup.py:_flush():81] Configure stats pid to 15014
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/2026-05-23/N_2.5/wandb/run-20260523_172819-hcp9axm4/logs/debug.log
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/2026-05-23/N_2.5/wandb/run-20260523_172819-hcp9axm4/logs/debug-internal.log
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_init.py:init():844] calling init triggers
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 3, 'max_steps': None, 'batch_size': 8, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0001, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.5, 'load_balancing_N': 2.5, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': 150000, 'max_val_samples': None}, 'logging': {'log_interval': 10, 'save_interval': 3000, 'eval_interval': 1000, 'save_every_epoch': False, 'model_only_checkpoints': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'routing-evolution', 'run_name': 'routing_N2.5', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/2026-05-23/N_2.5'}, 'paths': {'output_dir': 'outputs/2026-05-23/N_2.5'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/routing_evolution_exp/train.py'}}
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_init.py:init():892] starting backend
+2026-05-23 17:28:20,009 INFO    MainThread:15014 [wandb_init.py:init():895] sending inform_init request
+2026-05-23 17:28:20,030 INFO    MainThread:15014 [wandb_init.py:init():903] backend started and connected
+2026-05-23 17:28:20,037 INFO    MainThread:15014 [wandb_init.py:init():973] updated telemetry
+2026-05-23 17:28:20,063 INFO    MainThread:15014 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-23 17:28:20,628 INFO    MainThread:15014 [wandb_init.py:init():1044] starting run threads in backend
+2026-05-23 17:28:20,879 INFO    MainThread:15014 [wandb_run.py:_console_start():2529] atexit reg
+2026-05-23 17:28:20,879 INFO    MainThread:15014 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-05-23 17:28:20,879 INFO    MainThread:15014 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-05-23 17:28:20,879 INFO    MainThread:15014 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-05-23 17:28:20,883 INFO    MainThread:15014 [wandb_init.py:init():1084] run started, returning control to user process
+2026-05-24 00:08:42,341 INFO    MainThread:15014 [wandb_run.py:_finish():2295] finishing run nikita/routing-evolution/hcp9axm4
+2026-05-24 00:08:42,342 INFO    MainThread:15014 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-05-24 00:08:42,342 INFO    MainThread:15014 [wandb_run.py:_restore():2476] restore
+2026-05-24 00:08:42,342 INFO    MainThread:15014 [wandb_run.py:_restore():2482] restore done
+2026-05-24 00:08:43,394 INFO    MainThread:15014 [wandb_run.py:_footer_sync_info():3870] logging synced files

routing_rerun/N_2.5/wandb/run-20260523_172658-l6q9x03b/files/code/routing_evolution_exp/train.py

@@ -0,0 +1,259 @@
+"""
+Routing Evolution Experiment: H-Net fine-tune с разными load_balancing_N.
+
+Цель: изучить как routing module (W_q, W_k) эволюционирует при разном давлении сжатия.
+
+Использование:
+    # Запуск с конкретным N
+    accelerate launch --config_file accelerate_config.yaml \
+        train.py training.load_balancing_N=6.0 tracking.run_name=routing_N6.0
+
+    # Запуск всех N через run_all.sh
+    bash run_all.sh
+"""
+
+import math
+import os
+from pathlib import Path
+
+import torch
+import hydra
+from hydra.core.hydra_config import HydraConfig
+from omegaconf import DictConfig, OmegaConf
+from accelerate import Accelerator
+from accelerate.utils import set_seed as accelerate_set_seed
+
+# HNet imports
+from hnet.load_utils import load_from_pretrained, load_from_config
+from hnet.utils.tokenizers import ByteTokenizer
+from hnet.utils.train import group_params
+
+# Ensure repo root is on sys.path (needed when running from subdirectory)
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+# Shared training library
+from training_lib.utils import log_message
+from training_lib.checkpointing import save_checkpoint, load_checkpoint
+from training_lib.schedulers import get_lr_scheduler
+from training_lib.tracking import init_tracking, finish_tracking
+from training_lib.hnet.train_loop import train_epoch
+from training_lib.hnet.data import create_dataloaders
+
+
+def save_routing_weights(model, step, output_dir):
+    """Сохраняет только routing W_q/W_k для быстрого анализа weight evolution."""
+    routing_dir = output_dir / "routing_weights"
+    routing_dir.mkdir(parents=True, exist_ok=True)
+
+    state = model.state_dict()
+    routing_keys = [k for k in state if "routing_module" in k and "proj_layer" in k]
+    routing_weights = {k: state[k].cpu() for k in routing_keys}
+    torch.save(routing_weights, routing_dir / f"routing_step_{step}.pt")
+
+
+def on_checkpoint_callback(model, step, output_dir):
+    """Callback для train_epoch: сохраняет routing weights при каждом checkpoint.
+
+    Модель уже unwrapped в train_loop перед вызовом callback.
+    """
+    save_routing_weights(model, step, output_dir)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="config")
+def main(cfg: DictConfig):
+    """Тренировка H-Net с фиксированным числом шагов для анализа routing evolution."""
+
+    # === Accelerator Setup ===
+    mixed_precision = "bf16" if cfg.training.use_amp else "no"
+
+    accelerator = Accelerator(
+        mixed_precision=mixed_precision,
+        gradient_accumulation_steps=cfg.training.gradient_accumulation_steps,
+    )
+
+    accelerate_set_seed(cfg.seed)
+
+    if cfg.paths.output_dir is None:
+        cfg.paths.output_dir = HydraConfig.get().runtime.output_dir
+
+    OmegaConf.resolve(cfg)
+
+    log_message(
+        f"CUDA_VISIBLE_DEVICES: {os.environ.get('CUDA_VISIBLE_DEVICES', 'not set')}",
+        cfg,
+        accelerator,
+    )
+    log_message(f"Number of processes: {accelerator.num_processes}", cfg, accelerator)
+    log_message(f"Mixed precision: {mixed_precision}", cfg, accelerator)
+
+    log_message("=" * 60, cfg, accelerator)
+    log_message(
+        f"Routing Evolution Experiment | N={cfg.training.load_balancing_N}",
+        cfg,
+        accelerator,
+    )
+    log_message("=" * 60, cfg, accelerator)
+    log_message(f"Config:\n{OmegaConf.to_yaml(cfg)}", cfg, accelerator)
+
+    # === Experiment Tracking Init ===
+    init_tracking(cfg, accelerator)
+
+    # === Tokenizer ===
+    tokenizer = ByteTokenizer()
+
+    # === Model ===
+    log_message("Loading model...", cfg, accelerator)
+    if cfg.model.checkpoint_path:
+        model = load_from_pretrained(
+            model_path=cfg.model.checkpoint_path,
+            model_config_path=cfg.model.config_path,
+        )
+        log_message(f"Loaded pretrained: {cfg.model.checkpoint_path}", cfg, accelerator)
+    else:
+        model = load_from_config(
+            model_config_path=cfg.model.config_path,
+            device="cpu",
+        )
+        model.init_weights()
+        log_message("Initialized from scratch", cfg, accelerator)
+
+    model.train()
+
+    lr_multiplier = list(cfg.training.lr_multiplier)
+    model.apply_lr_multiplier(lr_multiplier)
+    log_message(f"Applied LR multipliers: {lr_multiplier}", cfg, accelerator)
+
+    if cfg.training.warmup_model:
+        log_message("Warming up model...", cfg, accelerator)
+        model = model.to(accelerator.device)
+        model.warmup(verbose=accelerator.is_main_process)
+
+    # Save initial routing weights (step 0)
+    if accelerator.is_main_process:
+        output_dir = Path(cfg.paths.output_dir)
+        save_routing_weights(model, 0, output_dir)
+
+    total_params = sum(p.numel() for p in model.parameters())
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    log_message(f"Total params: {total_params:,}", cfg, accelerator)
+    log_message(f"Trainable params: {trainable_params:,}", cfg, accelerator)
+
+    # === Data ===
+    log_message("Creating dataloaders...", cfg, accelerator)
+    dataloaders = create_dataloaders(cfg, tokenizer)
+
+    train_dataloader = dataloaders["train"]
+    val_dataloader = dataloaders.get("validation", None)
+
+    log_message(
+        f"Train dataset size: {len(train_dataloader.dataset)} "
+        f"(max_train_samples={cfg.data.max_train_samples}) | "
+        f"Epochs: {cfg.training.epochs}",
+        cfg,
+        accelerator,
+    )
+
+    # === Optimizer ===
+    param_groups = group_params(model)
+
+    for group in param_groups:
+        if "lr" not in group:
+            group["lr"] = cfg.training.lr
+        else:
+            group["lr"] = cfg.training.lr * group.get("lr_multiplier", 1.0)
+        if "weight_decay" not in group:
+            group["weight_decay"] = cfg.training.weight_decay
+
+    optimizer = torch.optim.AdamW(
+        param_groups,
+        lr=cfg.training.lr,
+        betas=tuple(cfg.training.betas),
+        eps=cfg.training.eps,
+    )
+
+    # === Scheduler ===
+    steps_per_epoch = math.ceil(len(train_dataloader) / accelerator.num_processes)
+    max_steps = (
+        cfg.training.epochs
+        * steps_per_epoch
+        // cfg.training.gradient_accumulation_steps
+    )
+    scheduler = get_lr_scheduler(optimizer, cfg, max_steps)
+
+    log_message(
+        f"Max steps: {max_steps}, Steps per epoch: {steps_per_epoch}",
+        cfg,
+        accelerator,
+    )
+
+    # === Accelerate Prepare ===
+    if val_dataloader is not None:
+        model, optimizer, train_dataloader, val_dataloader, scheduler = (
+            accelerator.prepare(
+                model, optimizer, train_dataloader, val_dataloader, scheduler
+            )
+        )
+    else:
+        model, optimizer, train_dataloader, scheduler = accelerator.prepare(
+            model, optimizer, train_dataloader, scheduler
+        )
+
+    # === Training Loop ===
+    log_message("Starting training...", cfg, accelerator)
+
+    global_step = 0
+    best_val_loss = float("inf")
+    epoch = 0
+
+    num_epochs = cfg.training.epochs
+
+    try:
+        for epoch in range(1, num_epochs + 1):
+            log_message(f"\n{'=' * 60}", cfg, accelerator)
+            log_message(
+                f"EPOCH {epoch}/{num_epochs} (step {global_step})", cfg, accelerator
+            )
+            log_message(f"{'=' * 60}", cfg, accelerator)
+
+            global_step, best_val_loss = train_epoch(
+                model=model,
+                dataloader=train_dataloader,
+                optimizer=optimizer,
+                scheduler=scheduler,
+                cfg=cfg,
+                epoch=epoch,
+                global_step=global_step,
+                accelerator=accelerator,
+                val_dataloader=val_dataloader,
+                best_val_loss=best_val_loss,
+                max_steps=max_steps,
+                on_checkpoint=on_checkpoint_callback,
+            )
+
+    except KeyboardInterrupt:
+        log_message("Training interrupted by user", cfg, accelerator)
+        save_checkpoint(
+            model, optimizer, scheduler, global_step, epoch, cfg, accelerator,
+            model_only=True,
+        )
+
+    # === Final Save ===
+    log_message("\nTraining completed!", cfg, accelerator)
+
+    if accelerator.is_main_process:
+        final_model_path = Path(cfg.paths.output_dir) / "model_final.pt"
+        unwrapped_model = accelerator.unwrap_model(model)
+        torch.save(unwrapped_model.state_dict(), final_model_path)
+        log_message(f"Final model: {final_model_path}", cfg, accelerator)
+
+        # Save final routing weights
+        save_routing_weights(unwrapped_model, global_step, Path(cfg.paths.output_dir))
+
+    accelerator.wait_for_everyone()
+    accelerator.end_training()
+    finish_tracking()
+
+
+if __name__ == "__main__":
+    main()

routing_rerun/N_2.5/wandb/run-20260523_172658-l6q9x03b/files/config.yaml

@@ -0,0 +1,132 @@
+_wandb:
+    value:
+        cli_version: 0.24.0
+        code_path: code/routing_evolution_exp/train.py
+        e:
+            xele1blpla9o3dqe7kdco4df3hrq09ni:
+                args:
+                    - training.load_balancing_N=2.5
+                    - tracking.run_name=routing_N2.5
+                codePath: routing_evolution_exp/train.py
+                codePathLocal: train.py
+                cpu_count: 36
+                cpu_count_logical: 72
+                cudaVersion: "13.2"
+                disk:
+                    /:
+                        total: "234075717632"
+                        used: "19049525248"
+                email: nikita@local.ru
+                executable: /venv/bytellm/bin/python
+                git:
+                    commit: 15d97dd48d00266a0b8f634920ab99cb7547da91
+                    remote: https://github.com/naryst/byte-llms-code.git
+                gpu: NVIDIA A100 80GB PCIe
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100 80GB PCIe
+                      uuid: GPU-ec9764c4-224a-9508-002f-de7762d78498
+                host: 1a023b226280
+                memory:
+                    total: "404274495488"
+                os: Linux-6.8.0-117-generic-x86_64-with-glibc2.39
+                program: /workspace/byte-llms-code/routing_evolution_exp/train.py
+                python: CPython 3.12.0
+                root: outputs/2026-05-23/N_2.5
+                startedAt: "2026-05-23T17:26:58.115428Z"
+                writerId: xele1blpla9o3dqe7kdco4df3hrq09ni
+        m: []
+        python_version: 3.12.0
+        t:
+            "1":
+                - 1
+                - 11
+                - 49
+                - 50
+                - 51
+                - 71
+                - 105
+            "2":
+                - 1
+                - 11
+                - 49
+                - 50
+                - 51
+                - 71
+                - 105
+            "3":
+                - 13
+                - 16
+            "4": 3.12.0
+            "5": 0.24.0
+            "6": 4.57.6
+            "12": 0.24.0
+            "13": linux-x86_64
+data:
+    value:
+        max_context_len: 4096
+        max_target_len: 256
+        max_train_samples: 150000
+        max_val_samples: null
+        num_workers: 0
+        path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+        pin_memory: true
+device:
+    value: cuda
+logging:
+    value:
+        eval_interval: 1000
+        log_interval: 10
+        model_only_checkpoints: true
+        save_every_epoch: false
+        save_interval: 3000
+model:
+    value:
+        checkpoint_path: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+        config_path: /workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json
+paths:
+    value:
+        output_dir: outputs/2026-05-23/N_2.5
+seed:
+    value: 42
+tracking:
+    value:
+        backend: wandb
+        base_url: https://wandb.platun0v.ru
+        enabled: true
+        entity: null
+        local_dir: outputs/2026-05-23/N_2.5
+        project: routing-evolution
+        run_name: routing_N2.5
+training:
+    value:
+        batch_size: 8
+        betas:
+            - 0.9
+            - 0.95
+        decay_ratio: 0.2
+        epochs: 3
+        eps: 1e-08
+        eval_batch_size: 24
+        gradient_accumulation_steps: 4
+        load_balancing_N: 2.5
+        load_balancing_weight: 0.5
+        lr: 0.0001
+        lr_multiplier:
+            - 2
+            - 1.5
+            - 1
+        lr_scheduler: wsd
+        max_grad_norm: 1
+        max_steps: null
+        min_lr_ratio: 0.1
+        resume: false
+        resume_checkpoint: null
+        use_amp: true
+        warmup_model: true
+        warmup_ratio: 0.1
+        warmup_steps: 100
+        weight_decay: 0.1

routing_rerun/N_2.5/wandb/run-20260523_172658-l6q9x03b/files/output.log

@@ -0,0 +1,108 @@
+[2026-05-23 17:26:59] Loading model...
+[2026-05-23 17:27:05] Loaded pretrained: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+[2026-05-23 17:27:05] Applied LR multipliers: [2.0, 1.5, 1.0]
+[2026-05-23 17:27:05] Warming up model...
+[WARMUP] Starting warmup (compiling Triton kernels)...
+Traceback (most recent call last):
+  File "/workspace/byte-llms-code/routing_evolution_exp/train.py", line 259, in <module>
+    main()
+  File "/venv/bytellm/lib/python3.12/site-packages/hydra/main.py", line 94, in decorated_main
+    _run_hydra(
+  File "/venv/bytellm/lib/python3.12/site-packages/hydra/_internal/utils.py", line 394, in _run_hydra
+    _run_app(
+  File "/venv/bytellm/lib/python3.12/site-packages/hydra/_internal/utils.py", line 457, in _run_app
+    run_and_report(
+  File "/venv/bytellm/lib/python3.12/site-packages/hydra/_internal/utils.py", line 220, in run_and_report
+    return func()
+           ^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/hydra/_internal/utils.py", line 458, in <lambda>
+    lambda: hydra.run(
+            ^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/hydra/_internal/hydra.py", line 119, in run
+    ret = run_job(
+          ^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/hydra/core/utils.py", line 186, in run_job
+    ret.return_value = task_function(task_cfg)
+                       ^^^^^^^^^^^^^^^^^^^^^^^
+  File "/workspace/byte-llms-code/routing_evolution_exp/train.py", line 130, in main
+    model.warmup(verbose=accelerator.is_main_process)
+  File "/workspace/byte-llms-code/hnet_project/hnet/models/mixer_seq.py", line 126, in warmup
+    output = self.forward(dummy_input)
+             ^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/workspace/byte-llms-code/hnet_project/hnet/models/mixer_seq.py", line 181, in forward
+    hidden_states, bpred_output = self.backbone(
+                                  ^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/workspace/byte-llms-code/hnet_project/hnet/models/hnet.py", line 333, in forward
+    hidden_states = self.encoder(
+                    ^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/workspace/byte-llms-code/hnet_project/hnet/modules/isotropic.py", line 180, in forward
+    hidden_states, residual = layer(
+                              ^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/workspace/byte-llms-code/hnet_project/hnet/modules/block.py", line 226, in forward
+    hidden_states = self.mixer(
+                    ^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1739, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1750, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/mamba_ssm/modules/mamba2.py", line 185, in forward
+    out = mamba_split_conv1d_scan_combined(
+          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/mamba_ssm/ops/triton/ssd_combined.py", line 930, in mamba_split_conv1d_scan_combined
+    return MambaSplitConv1dScanCombinedFn.apply(zxbcdt, conv1d_weight, conv1d_bias, dt_bias, A, D, chunk_size, initial_states, seq_idx, dt_limit, return_final_states, activation, rmsnorm_weight, rmsnorm_eps, outproj_weight, outproj_bias, headdim, ngroups, norm_before_gate)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/torch/autograd/function.py", line 575, in apply
+    return super().apply(*args, **kwargs)  # type: ignore[misc]
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/torch/amp/autocast_mode.py", line 503, in decorate_fwd
+    return fwd(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/mamba_ssm/ops/triton/ssd_combined.py", line 795, in forward
+    out_x, _, dt_out, dA_cumsum, states, final_states = _mamba_chunk_scan_combined_fwd(x, dt, A, B, C, chunk_size=chunk_size, D=D, z=None, dt_bias=dt_bias, initial_states=initial_states, seq_idx=seq_idx, dt_softplus=True, dt_limit=dt_limit)
+                                                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/mamba_ssm/ops/triton/ssd_combined.py", line 324, in _mamba_chunk_scan_combined_fwd
+    out, out_x = _chunk_scan_fwd(CB, x, dt, dA_cumsum, C, states, D=D, z=z, seq_idx=seq_idx)
+                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/mamba_ssm/ops/triton/ssd_chunk_scan.py", line 1257, in _chunk_scan_fwd
+    _chunk_scan_fwd_kernel[grid](
+  File "/venv/bytellm/lib/python3.12/site-packages/triton/runtime/jit.py", line 330, in <lambda>
+    return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs)
+                                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/triton/runtime/autotuner.py", line 186, in run
+    timings = {config: self._bench(*args, config=config, **kwargs) for config in pruned_configs}
+                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/triton/runtime/autotuner.py", line 166, in _bench
+    return self.do_bench(kernel_call, quantiles=(0.5, 0.2, 0.8))
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/venv/bytellm/lib/python3.12/site-packages/triton/testing.py", line 140, in do_bench
+    fn()
+  File "/venv/bytellm/lib/python3.12/site-packages/triton/runtime/autotuner.py", line 152, in kernel_call
+    self.fn.run(
+  File "/venv/bytellm/lib/python3.12/site-packages/triton/runtime/jit.py", line 580, in run
+    bound_args, sig_and_spec, constexpr_vals, non_constexpr_vals, excess_kwargs = self.binder(*args, **kwargs)
+                                                                                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "<string>", line 2, in dynamic_func
+  File "/venv/bytellm/lib/python3.12/site-packages/triton/runtime/jit.py", line 294, in mangle_type
+    def mangle_type(arg, is_const=False):
+
+KeyboardInterrupt

routing_rerun/N_2.5/wandb/run-20260523_172658-l6q9x03b/files/requirements.txt

@@ -0,0 +1,245 @@
+setuptools==78.1.1
+wheel==0.45.1
+pip==25.2
+webencodings==0.5.1
+triton==3.2.0
+pytz==2025.2
+pydub==0.25.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.6.2
+mpmath==1.3.0
+ipython-genutils==0.2.0
+fastjsonschema==2.21.2
+brotli==1.2.0
+antlr4-python3-runtime==4.9.3
+xxhash==3.6.0
+widgetsnbextension==4.0.14
+websocket-client==1.9.0
+webcolors==24.11.1
+wcwidth==0.2.14
+urllib3==2.5.0
+uri-template==1.3.0
+tzdata==2025.2
+typing_extensions==4.15.0
+types-python-dateutil==2.9.0.20251008
+traitlets==5.14.3
+tqdm==4.67.1
+tornado==6.5.2
+tomlkit==0.13.3
+tinycss2==1.4.0
+tabulate==0.9.0
+sympy==1.13.1
+soupsieve==2.8
+sniffio==1.3.1
+smmap==5.0.2
+six==1.17.0
+shellingham==1.5.4
+Send2Trash==1.8.3
+semantic-version==2.10.0
+safetensors==0.6.2
+rpds-py==0.27.1
+rfc3986-validator==0.1.1
+regex==2025.9.18
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-multipart==0.0.22
+python-json-logger==4.0.0
+python-dotenv==1.2.1
+pyparsing==3.2.5
+PyJWT==2.8.0
+Pygments==2.19.2
+pycparser==2.23
+pyarrow==22.0.0
+psutil==7.1.0
+protobuf==6.33.4
+propcache==0.4.1
+prometheus_client==0.23.1
+portalocker==3.2.0
+platformdirs==4.5.0
+pillow==11.3.0
+pexpect==4.9.0
+pathspec==1.0.4
+parso==0.8.5
+pandocfilters==1.5.1
+packaging==25.0
+orjson==3.11.6
+opt_einsum==3.4.0
+nvidia-nvtx-cu12==12.4.127
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nccl-cu12==2.21.5
+nvidia-curand-cu12==10.3.5.147
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cufft-cu12==11.2.1.3
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cublas-cu12==12.4.5.8
+numpy==2.3.3
+ninja==1.13.0
+networkx==3.5
+nest-asyncio==1.6.0
+narwhals==2.15.0
+mypy_extensions==1.1.0
+multidict==6.7.0
+mistune==3.1.4
+mdurl==0.1.2
+MarkupSafe==3.0.3
+lxml==6.0.2
+librt==0.8.0
+lark==1.3.0
+kiwisolver==1.4.9
+jupyterlab_widgets==3.0.15
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.12.1
+itsdangerous==2.2.0
+idna==3.10
+hf-xet==1.1.10
+h11==0.16.0
+groovy==0.1.2
+fsspec==2025.9.0
+frozenlist==1.8.0
+fqdn==1.5.1
+fonttools==4.60.1
+filelock==3.19.1
+ffmpy==1.0.0
+executing==2.2.1
+einops==0.8.1
+dill==0.4.0
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.17
+dacite==1.9.2
+cycler==0.12.1
+comm==0.2.3
+colorama==0.4.6
+click==8.3.1
+charset-normalizer==3.4.3
+certifi==2025.10.5
+bleach==6.2.0
+babel==2.17.0
+attrs==25.4.0
+async-lru==2.0.5
+asttokens==3.0.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+aiofiles==24.1.0
+yarl==1.22.0
+uvicorn==0.40.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+sentry-sdk==2.50.0
+scipy==1.17.0
+sacrebleu==2.6.0
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+reportlab==4.4.9
+referencing==0.36.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+plotly==6.5.2
+pathlib2==2.3.7.post1
+orderedmultidict==1.0.2
+optree==0.17.0
+omegaconf==2.3.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+mypy==1.19.1
+multiprocess==0.70.16
+matplotlib-inline==0.1.7
+markdown-it-py==4.0.0
+jupyter_core==5.8.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+httpcore==1.0.9
+gitdb==4.0.12
+ftfy==6.3.1
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.2
+anyio==4.11.0
+aiosignal==1.4.0
+starlette==0.50.0
+rich==14.2.0
+pydantic==2.12.5
+pandas==2.3.3
+nvidia-cusolver-cu12==11.6.1.9
+matplotlib==3.10.7
+jupyter_server_terminals==0.5.3
+jupyter_client==8.6.3
+jsonschema-specifications==2025.9.1
+ipython==9.6.0
+hydra-core==1.3.2
+huggingface-hub==0.35.3
+httpx==0.28.1
+GitPython==3.1.46
+furl==2.1.4
+cryptography==46.0.4
+arrow==1.3.0
+argon2-cffi-bindings==25.1.0
+aiohttp==3.13.1
+wandb==0.24.0
+typer==0.21.1
+torch==2.6.0
+tokenizers==0.22.1
+seaborn==0.13.2
+safehttpx==0.1.7
+jsonschema==4.25.1
+joypy==0.2.6
+isoduration==20.11.0
+ipywidgets==8.1.7
+ipykernel==6.30.1
+gradio_client==2.0.3
+fastapi==0.128.0
+Authlib==1.6.6
+argon2-cffi==25.1.0
+transformers==4.57.6
+nbformat==5.10.4
+mlstm_kernels==2.0.2
+jupyter-console==6.6.3
+gradio==6.5.1
+datasets==4.3.0
+clearml==1.16.4
+accelerate==1.10.1
+xlstm==2.0.4
+nbclient==0.10.2
+jupyter-events==0.12.0
+trackio==0.15.0
+nbconvert==7.16.6
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.27.3
+jupyter-lsp==2.3.0
+nbclassic==1.3.3
+jupyterlab==4.4.9
+notebook==7.4.7
+jupyter_contrib_core==0.4.2
+jupyter==1.1.1
+jupyter_nbextensions_configurator==0.6.4
+causal-conv1d==1.5.0.post8
+flash_attn==2.7.4.post1
+mamba-ssm==2.2.4
+hnet==0.0.1
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

routing_rerun/N_2.5/wandb/run-20260523_172658-l6q9x03b/files/wandb-metadata.json

@@ -0,0 +1,44 @@
+{
+  "os": "Linux-6.8.0-117-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.0",
+  "startedAt": "2026-05-23T17:26:58.115428Z",
+  "args": [
+    "training.load_balancing_N=2.5",
+    "tracking.run_name=routing_N2.5"
+  ],
+  "program": "/workspace/byte-llms-code/routing_evolution_exp/train.py",
+  "codePath": "routing_evolution_exp/train.py",
+  "codePathLocal": "train.py",
+  "git": {
+    "remote": "https://github.com/naryst/byte-llms-code.git",
+    "commit": "15d97dd48d00266a0b8f634920ab99cb7547da91"
+  },
+  "email": "nikita@local.ru",
+  "root": "outputs/2026-05-23/N_2.5",
+  "host": "1a023b226280",
+  "executable": "/venv/bytellm/bin/python",
+  "cpu_count": 36,
+  "cpu_count_logical": 72,
+  "gpu": "NVIDIA A100 80GB PCIe",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "234075717632",
+      "used": "19049525248"
+    }
+  },
+  "memory": {
+    "total": "404274495488"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100 80GB PCIe",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ec9764c4-224a-9508-002f-de7762d78498"
+    }
+  ],
+  "cudaVersion": "13.2",
+  "writerId": "xele1blpla9o3dqe7kdco4df3hrq09ni"
+}

routing_rerun/N_2.5/wandb/run-20260523_172658-l6q9x03b/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":43},"_runtime":43}

routing_rerun/N_2.5/wandb/run-20260523_172658-l6q9x03b/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2026-05-23T17:26:58.232625454Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpf4ecvb2u/port-14607.txt","pid":14607,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-05-23T17:26:58.233976439Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":14607}
+{"time":"2026-05-23T17:26:58.233282033Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-14607-14617-1099789617/socket","Net":"unix"}}
+{"time":"2026-05-23T17:26:58.419094136Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-05-23T17:26:58.44527049Z","level":"INFO","msg":"handleInformInit: received","streamId":"l6q9x03b","id":"1(@)"}
+{"time":"2026-05-23T17:26:58.885396624Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"l6q9x03b","id":"1(@)"}
+{"time":"2026-05-23T17:27:42.947784883Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-05-23T17:27:42.947914409Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-05-23T17:27:42.947902597Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-05-23T17:27:42.948232086Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-05-23T17:27:42.948255177Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-14607-14617-1099789617/socket","Net":"unix"}}
+{"time":"2026-05-23T17:27:43.504784979Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-05-23T17:27:43.504842499Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-05-23T17:27:43.504876254Z","level":"INFO","msg":"server is closed"}

routing_rerun/N_2.5/wandb/run-20260523_172658-l6q9x03b/logs/debug-internal.log

@@ -0,0 +1,14 @@
+{"time":"2026-05-23T17:26:58.445481693Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-05-23T17:26:58.885208159Z","level":"INFO","msg":"stream: created new stream","id":"l6q9x03b"}
+{"time":"2026-05-23T17:26:58.885289947Z","level":"INFO","msg":"handler: started","stream_id":"l6q9x03b"}
+{"time":"2026-05-23T17:26:58.885385406Z","level":"INFO","msg":"stream: started","id":"l6q9x03b"}
+{"time":"2026-05-23T17:26:58.885419994Z","level":"INFO","msg":"writer: started","stream_id":"l6q9x03b"}
+{"time":"2026-05-23T17:26:58.88542392Z","level":"INFO","msg":"sender: started","stream_id":"l6q9x03b"}
+{"time":"2026-05-23T17:26:59.024082436Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-23T17:27:42.946648107Z","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":68}
+{"time":"2026-05-23T17:27:42.947906186Z","level":"INFO","msg":"stream: closing","id":"l6q9x03b"}
+{"time":"2026-05-23T17:27:42.955515536Z","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":34}
+{"time":"2026-05-23T17:27:43.302861641Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-23T17:27:43.500084989Z","level":"INFO","msg":"handler: closed","stream_id":"l6q9x03b"}
+{"time":"2026-05-23T17:27:43.500251865Z","level":"INFO","msg":"sender: closed","stream_id":"l6q9x03b"}
+{"time":"2026-05-23T17:27:43.500269359Z","level":"INFO","msg":"stream: closed","id":"l6q9x03b"}

routing_rerun/N_2.5/wandb/run-20260523_172658-l6q9x03b/logs/debug.log

@@ -0,0 +1,21 @@
+2026-05-23 17:26:58,117 INFO    MainThread:14607 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-05-23 17:26:58,117 INFO    MainThread:14607 [wandb_setup.py:_flush():81] Configure stats pid to 14607
+2026-05-23 17:26:58,117 INFO    MainThread:14607 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-23 17:26:58,117 INFO    MainThread:14607 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/2026-05-23/N_2.5/wandb/run-20260523_172658-l6q9x03b/logs/debug.log
+2026-05-23 17:26:58,117 INFO    MainThread:14607 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/2026-05-23/N_2.5/wandb/run-20260523_172658-l6q9x03b/logs/debug-internal.log
+2026-05-23 17:26:58,117 INFO    MainThread:14607 [wandb_init.py:init():844] calling init triggers
+2026-05-23 17:26:58,118 INFO    MainThread:14607 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 3, 'max_steps': None, 'batch_size': 8, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0001, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.5, 'load_balancing_N': 2.5, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': 150000, 'max_val_samples': None}, 'logging': {'log_interval': 10, 'save_interval': 3000, 'eval_interval': 1000, 'save_every_epoch': False, 'model_only_checkpoints': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'routing-evolution', 'run_name': 'routing_N2.5', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/2026-05-23/N_2.5'}, 'paths': {'output_dir': 'outputs/2026-05-23/N_2.5'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/routing_evolution_exp/train.py'}}
+2026-05-23 17:26:58,118 INFO    MainThread:14607 [wandb_init.py:init():892] starting backend
+2026-05-23 17:26:58,419 INFO    MainThread:14607 [wandb_init.py:init():895] sending inform_init request
+2026-05-23 17:26:58,441 INFO    MainThread:14607 [wandb_init.py:init():903] backend started and connected
+2026-05-23 17:26:58,447 INFO    MainThread:14607 [wandb_init.py:init():973] updated telemetry
+2026-05-23 17:26:58,471 INFO    MainThread:14607 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-23 17:26:59,022 INFO    MainThread:14607 [wandb_init.py:init():1044] starting run threads in backend
+2026-05-23 17:26:59,270 INFO    MainThread:14607 [wandb_run.py:_console_start():2529] atexit reg
+2026-05-23 17:26:59,270 INFO    MainThread:14607 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-05-23 17:26:59,271 INFO    MainThread:14607 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-05-23 17:26:59,271 INFO    MainThread:14607 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-05-23 17:26:59,274 INFO    MainThread:14607 [wandb_init.py:init():1084] run started, returning control to user process
+2026-05-23 17:27:42,948 INFO    wandb-AsyncioManager-main:14607 [service_client.py:_forward_responses():80] Reached EOF.
+2026-05-23 17:27:42,949 INFO    wandb-AsyncioManager-main:14607 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

routing_rerun/N_2.5/wandb/run-20260523_172819-hcp9axm4/files/code/routing_evolution_exp/train.py

@@ -0,0 +1,259 @@
+"""
+Routing Evolution Experiment: H-Net fine-tune с разными load_balancing_N.
+
+Цель: изучить как routing module (W_q, W_k) эволюционирует при разном давлении сжатия.
+
+Использование:
+    # Запуск с конкретным N
+    accelerate launch --config_file accelerate_config.yaml \
+        train.py training.load_balancing_N=6.0 tracking.run_name=routing_N6.0
+
+    # Запуск всех N через run_all.sh
+    bash run_all.sh
+"""
+
+import math
+import os
+from pathlib import Path
+
+import torch
+import hydra
+from hydra.core.hydra_config import HydraConfig
+from omegaconf import DictConfig, OmegaConf
+from accelerate import Accelerator
+from accelerate.utils import set_seed as accelerate_set_seed
+
+# HNet imports
+from hnet.load_utils import load_from_pretrained, load_from_config
+from hnet.utils.tokenizers import ByteTokenizer
+from hnet.utils.train import group_params
+
+# Ensure repo root is on sys.path (needed when running from subdirectory)
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+# Shared training library
+from training_lib.utils import log_message
+from training_lib.checkpointing import save_checkpoint, load_checkpoint
+from training_lib.schedulers import get_lr_scheduler
+from training_lib.tracking import init_tracking, finish_tracking
+from training_lib.hnet.train_loop import train_epoch
+from training_lib.hnet.data import create_dataloaders
+
+
+def save_routing_weights(model, step, output_dir):
+    """Сохраняет только routing W_q/W_k для быстрого анализа weight evolution."""
+    routing_dir = output_dir / "routing_weights"
+    routing_dir.mkdir(parents=True, exist_ok=True)
+
+    state = model.state_dict()
+    routing_keys = [k for k in state if "routing_module" in k and "proj_layer" in k]
+    routing_weights = {k: state[k].cpu() for k in routing_keys}
+    torch.save(routing_weights, routing_dir / f"routing_step_{step}.pt")
+
+
+def on_checkpoint_callback(model, step, output_dir):
+    """Callback для train_epoch: сохраняет routing weights при каждом checkpoint.
+
+    Модель уже unwrapped в train_loop перед вызовом callback.
+    """
+    save_routing_weights(model, step, output_dir)
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="config")
+def main(cfg: DictConfig):
+    """Тренировка H-Net с фиксированным числом шагов для анализа routing evolution."""
+
+    # === Accelerator Setup ===
+    mixed_precision = "bf16" if cfg.training.use_amp else "no"
+
+    accelerator = Accelerator(
+        mixed_precision=mixed_precision,
+        gradient_accumulation_steps=cfg.training.gradient_accumulation_steps,
+    )
+
+    accelerate_set_seed(cfg.seed)
+
+    if cfg.paths.output_dir is None:
+        cfg.paths.output_dir = HydraConfig.get().runtime.output_dir
+
+    OmegaConf.resolve(cfg)
+
+    log_message(
+        f"CUDA_VISIBLE_DEVICES: {os.environ.get('CUDA_VISIBLE_DEVICES', 'not set')}",
+        cfg,
+        accelerator,
+    )
+    log_message(f"Number of processes: {accelerator.num_processes}", cfg, accelerator)
+    log_message(f"Mixed precision: {mixed_precision}", cfg, accelerator)
+
+    log_message("=" * 60, cfg, accelerator)
+    log_message(
+        f"Routing Evolution Experiment | N={cfg.training.load_balancing_N}",
+        cfg,
+        accelerator,
+    )
+    log_message("=" * 60, cfg, accelerator)
+    log_message(f"Config:\n{OmegaConf.to_yaml(cfg)}", cfg, accelerator)
+
+    # === Experiment Tracking Init ===
+    init_tracking(cfg, accelerator)
+
+    # === Tokenizer ===
+    tokenizer = ByteTokenizer()
+
+    # === Model ===
+    log_message("Loading model...", cfg, accelerator)
+    if cfg.model.checkpoint_path:
+        model = load_from_pretrained(
+            model_path=cfg.model.checkpoint_path,
+            model_config_path=cfg.model.config_path,
+        )
+        log_message(f"Loaded pretrained: {cfg.model.checkpoint_path}", cfg, accelerator)
+    else:
+        model = load_from_config(
+            model_config_path=cfg.model.config_path,
+            device="cpu",
+        )
+        model.init_weights()
+        log_message("Initialized from scratch", cfg, accelerator)
+
+    model.train()
+
+    lr_multiplier = list(cfg.training.lr_multiplier)
+    model.apply_lr_multiplier(lr_multiplier)
+    log_message(f"Applied LR multipliers: {lr_multiplier}", cfg, accelerator)
+
+    if cfg.training.warmup_model:
+        log_message("Warming up model...", cfg, accelerator)
+        model = model.to(accelerator.device)
+        model.warmup(verbose=accelerator.is_main_process)
+
+    # Save initial routing weights (step 0)
+    if accelerator.is_main_process:
+        output_dir = Path(cfg.paths.output_dir)
+        save_routing_weights(model, 0, output_dir)
+
+    total_params = sum(p.numel() for p in model.parameters())
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    log_message(f"Total params: {total_params:,}", cfg, accelerator)
+    log_message(f"Trainable params: {trainable_params:,}", cfg, accelerator)
+
+    # === Data ===
+    log_message("Creating dataloaders...", cfg, accelerator)
+    dataloaders = create_dataloaders(cfg, tokenizer)
+
+    train_dataloader = dataloaders["train"]
+    val_dataloader = dataloaders.get("validation", None)
+
+    log_message(
+        f"Train dataset size: {len(train_dataloader.dataset)} "
+        f"(max_train_samples={cfg.data.max_train_samples}) | "
+        f"Epochs: {cfg.training.epochs}",
+        cfg,
+        accelerator,
+    )
+
+    # === Optimizer ===
+    param_groups = group_params(model)
+
+    for group in param_groups:
+        if "lr" not in group:
+            group["lr"] = cfg.training.lr
+        else:
+            group["lr"] = cfg.training.lr * group.get("lr_multiplier", 1.0)
+        if "weight_decay" not in group:
+            group["weight_decay"] = cfg.training.weight_decay
+
+    optimizer = torch.optim.AdamW(
+        param_groups,
+        lr=cfg.training.lr,
+        betas=tuple(cfg.training.betas),
+        eps=cfg.training.eps,
+    )
+
+    # === Scheduler ===
+    steps_per_epoch = math.ceil(len(train_dataloader) / accelerator.num_processes)
+    max_steps = (
+        cfg.training.epochs
+        * steps_per_epoch
+        // cfg.training.gradient_accumulation_steps
+    )
+    scheduler = get_lr_scheduler(optimizer, cfg, max_steps)
+
+    log_message(
+        f"Max steps: {max_steps}, Steps per epoch: {steps_per_epoch}",
+        cfg,
+        accelerator,
+    )
+
+    # === Accelerate Prepare ===
+    if val_dataloader is not None:
+        model, optimizer, train_dataloader, val_dataloader, scheduler = (
+            accelerator.prepare(
+                model, optimizer, train_dataloader, val_dataloader, scheduler
+            )
+        )
+    else:
+        model, optimizer, train_dataloader, scheduler = accelerator.prepare(
+            model, optimizer, train_dataloader, scheduler
+        )
+
+    # === Training Loop ===
+    log_message("Starting training...", cfg, accelerator)
+
+    global_step = 0
+    best_val_loss = float("inf")
+    epoch = 0
+
+    num_epochs = cfg.training.epochs
+
+    try:
+        for epoch in range(1, num_epochs + 1):
+            log_message(f"\n{'=' * 60}", cfg, accelerator)
+            log_message(
+                f"EPOCH {epoch}/{num_epochs} (step {global_step})", cfg, accelerator
+            )
+            log_message(f"{'=' * 60}", cfg, accelerator)
+
+            global_step, best_val_loss = train_epoch(
+                model=model,
+                dataloader=train_dataloader,
+                optimizer=optimizer,
+                scheduler=scheduler,
+                cfg=cfg,
+                epoch=epoch,
+                global_step=global_step,
+                accelerator=accelerator,
+                val_dataloader=val_dataloader,
+                best_val_loss=best_val_loss,
+                max_steps=max_steps,
+                on_checkpoint=on_checkpoint_callback,
+            )
+
+    except KeyboardInterrupt:
+        log_message("Training interrupted by user", cfg, accelerator)
+        save_checkpoint(
+            model, optimizer, scheduler, global_step, epoch, cfg, accelerator,
+            model_only=True,
+        )
+
+    # === Final Save ===
+    log_message("\nTraining completed!", cfg, accelerator)
+
+    if accelerator.is_main_process:
+        final_model_path = Path(cfg.paths.output_dir) / "model_final.pt"
+        unwrapped_model = accelerator.unwrap_model(model)
+        torch.save(unwrapped_model.state_dict(), final_model_path)
+        log_message(f"Final model: {final_model_path}", cfg, accelerator)
+
+        # Save final routing weights
+        save_routing_weights(unwrapped_model, global_step, Path(cfg.paths.output_dir))
+
+    accelerator.wait_for_everyone()
+    accelerator.end_training()
+    finish_tracking()
+
+
+if __name__ == "__main__":
+    main()

routing_rerun/N_2.5/wandb/run-20260523_172819-hcp9axm4/files/config.yaml

@@ -0,0 +1,134 @@
+_wandb:
+    value:
+        cli_version: 0.24.0
+        code_path: code/routing_evolution_exp/train.py
+        e:
+            bnp9f95qneklhh3kico6ks4t4aw6iad3:
+                args:
+                    - training.load_balancing_N=2.5
+                    - tracking.run_name=routing_N2.5
+                codePath: routing_evolution_exp/train.py
+                codePathLocal: train.py
+                cpu_count: 36
+                cpu_count_logical: 72
+                cudaVersion: "13.2"
+                disk:
+                    /:
+                        total: "234075717632"
+                        used: "19065532416"
+                email: nikita@local.ru
+                executable: /venv/bytellm/bin/python
+                git:
+                    commit: 15d97dd48d00266a0b8f634920ab99cb7547da91
+                    remote: https://github.com/naryst/byte-llms-code.git
+                gpu: NVIDIA A100 80GB PCIe
+                gpu_count: 1
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100 80GB PCIe
+                      uuid: GPU-ec9764c4-224a-9508-002f-de7762d78498
+                host: 1a023b226280
+                memory:
+                    total: "404274495488"
+                os: Linux-6.8.0-117-generic-x86_64-with-glibc2.39
+                program: /workspace/byte-llms-code/routing_evolution_exp/train.py
+                python: CPython 3.12.0
+                root: outputs/2026-05-23/N_2.5
+                startedAt: "2026-05-23T17:28:19.719235Z"
+                writerId: bnp9f95qneklhh3kico6ks4t4aw6iad3
+        m: []
+        python_version: 3.12.0
+        t:
+            "1":
+                - 1
+                - 11
+                - 49
+                - 50
+                - 51
+                - 71
+                - 105
+            "2":
+                - 1
+                - 11
+                - 49
+                - 50
+                - 51
+                - 71
+                - 105
+            "3":
+                - 2
+                - 13
+                - 16
+                - 61
+            "4": 3.12.0
+            "5": 0.24.0
+            "6": 4.57.6
+            "12": 0.24.0
+            "13": linux-x86_64
+data:
+    value:
+        max_context_len: 4096
+        max_target_len: 256
+        max_train_samples: 150000
+        max_val_samples: null
+        num_workers: 0
+        path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+        pin_memory: true
+device:
+    value: cuda
+logging:
+    value:
+        eval_interval: 1000
+        log_interval: 10
+        model_only_checkpoints: true
+        save_every_epoch: false
+        save_interval: 3000
+model:
+    value:
+        checkpoint_path: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+        config_path: /workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json
+paths:
+    value:
+        output_dir: outputs/2026-05-23/N_2.5
+seed:
+    value: 42
+tracking:
+    value:
+        backend: wandb
+        base_url: https://wandb.platun0v.ru
+        enabled: true
+        entity: null
+        local_dir: outputs/2026-05-23/N_2.5
+        project: routing-evolution
+        run_name: routing_N2.5
+training:
+    value:
+        batch_size: 8
+        betas:
+            - 0.9
+            - 0.95
+        decay_ratio: 0.2
+        epochs: 3
+        eps: 1e-08
+        eval_batch_size: 24
+        gradient_accumulation_steps: 4
+        load_balancing_N: 2.5
+        load_balancing_weight: 0.5
+        lr: 0.0001
+        lr_multiplier:
+            - 2
+            - 1.5
+            - 1
+        lr_scheduler: wsd
+        max_grad_norm: 1
+        max_steps: null
+        min_lr_ratio: 0.1
+        resume: false
+        resume_checkpoint: null
+        use_amp: true
+        warmup_model: true
+        warmup_ratio: 0.1
+        warmup_steps: 100
+        weight_decay: 0.1

routing_rerun/N_2.5/wandb/run-20260523_172819-hcp9axm4/files/requirements.txt

@@ -0,0 +1,245 @@
+setuptools==78.1.1
+wheel==0.45.1
+pip==25.2
+webencodings==0.5.1
+triton==3.2.0
+pytz==2025.2
+pydub==0.25.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.6.2
+mpmath==1.3.0
+ipython-genutils==0.2.0
+fastjsonschema==2.21.2
+brotli==1.2.0
+antlr4-python3-runtime==4.9.3
+xxhash==3.6.0
+widgetsnbextension==4.0.14
+websocket-client==1.9.0
+webcolors==24.11.1
+wcwidth==0.2.14
+urllib3==2.5.0
+uri-template==1.3.0
+tzdata==2025.2
+typing_extensions==4.15.0
+types-python-dateutil==2.9.0.20251008
+traitlets==5.14.3
+tqdm==4.67.1
+tornado==6.5.2
+tomlkit==0.13.3
+tinycss2==1.4.0
+tabulate==0.9.0
+sympy==1.13.1
+soupsieve==2.8
+sniffio==1.3.1
+smmap==5.0.2
+six==1.17.0
+shellingham==1.5.4
+Send2Trash==1.8.3
+semantic-version==2.10.0
+safetensors==0.6.2
+rpds-py==0.27.1
+rfc3986-validator==0.1.1
+regex==2025.9.18
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-multipart==0.0.22
+python-json-logger==4.0.0
+python-dotenv==1.2.1
+pyparsing==3.2.5
+PyJWT==2.8.0
+Pygments==2.19.2
+pycparser==2.23
+pyarrow==22.0.0
+psutil==7.1.0
+protobuf==6.33.4
+propcache==0.4.1
+prometheus_client==0.23.1
+portalocker==3.2.0
+platformdirs==4.5.0
+pillow==11.3.0
+pexpect==4.9.0
+pathspec==1.0.4
+parso==0.8.5
+pandocfilters==1.5.1
+packaging==25.0
+orjson==3.11.6
+opt_einsum==3.4.0
+nvidia-nvtx-cu12==12.4.127
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nccl-cu12==2.21.5
+nvidia-curand-cu12==10.3.5.147
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cufft-cu12==11.2.1.3
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cublas-cu12==12.4.5.8
+numpy==2.3.3
+ninja==1.13.0
+networkx==3.5
+nest-asyncio==1.6.0
+narwhals==2.15.0
+mypy_extensions==1.1.0
+multidict==6.7.0
+mistune==3.1.4
+mdurl==0.1.2
+MarkupSafe==3.0.3
+lxml==6.0.2
+librt==0.8.0
+lark==1.3.0
+kiwisolver==1.4.9
+jupyterlab_widgets==3.0.15
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.12.1
+itsdangerous==2.2.0
+idna==3.10
+hf-xet==1.1.10
+h11==0.16.0
+groovy==0.1.2
+fsspec==2025.9.0
+frozenlist==1.8.0
+fqdn==1.5.1
+fonttools==4.60.1
+filelock==3.19.1
+ffmpy==1.0.0
+executing==2.2.1
+einops==0.8.1
+dill==0.4.0
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.17
+dacite==1.9.2
+cycler==0.12.1
+comm==0.2.3
+colorama==0.4.6
+click==8.3.1
+charset-normalizer==3.4.3
+certifi==2025.10.5
+bleach==6.2.0
+babel==2.17.0
+attrs==25.4.0
+async-lru==2.0.5
+asttokens==3.0.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+aiofiles==24.1.0
+yarl==1.22.0
+uvicorn==0.40.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+sentry-sdk==2.50.0
+scipy==1.17.0
+sacrebleu==2.6.0
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+reportlab==4.4.9
+referencing==0.36.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+plotly==6.5.2
+pathlib2==2.3.7.post1
+orderedmultidict==1.0.2
+optree==0.17.0
+omegaconf==2.3.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+mypy==1.19.1
+multiprocess==0.70.16
+matplotlib-inline==0.1.7
+markdown-it-py==4.0.0
+jupyter_core==5.8.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+httpcore==1.0.9
+gitdb==4.0.12
+ftfy==6.3.1
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.2
+anyio==4.11.0
+aiosignal==1.4.0
+starlette==0.50.0
+rich==14.2.0
+pydantic==2.12.5
+pandas==2.3.3
+nvidia-cusolver-cu12==11.6.1.9
+matplotlib==3.10.7
+jupyter_server_terminals==0.5.3
+jupyter_client==8.6.3
+jsonschema-specifications==2025.9.1
+ipython==9.6.0
+hydra-core==1.3.2
+huggingface-hub==0.35.3
+httpx==0.28.1
+GitPython==3.1.46
+furl==2.1.4
+cryptography==46.0.4
+arrow==1.3.0
+argon2-cffi-bindings==25.1.0
+aiohttp==3.13.1
+wandb==0.24.0
+typer==0.21.1
+torch==2.6.0
+tokenizers==0.22.1
+seaborn==0.13.2
+safehttpx==0.1.7
+jsonschema==4.25.1
+joypy==0.2.6
+isoduration==20.11.0
+ipywidgets==8.1.7
+ipykernel==6.30.1
+gradio_client==2.0.3
+fastapi==0.128.0
+Authlib==1.6.6
+argon2-cffi==25.1.0
+transformers==4.57.6
+nbformat==5.10.4
+mlstm_kernels==2.0.2
+jupyter-console==6.6.3
+gradio==6.5.1
+datasets==4.3.0
+clearml==1.16.4
+accelerate==1.10.1
+xlstm==2.0.4
+nbclient==0.10.2
+jupyter-events==0.12.0
+trackio==0.15.0
+nbconvert==7.16.6
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.27.3
+jupyter-lsp==2.3.0
+nbclassic==1.3.3
+jupyterlab==4.4.9
+notebook==7.4.7
+jupyter_contrib_core==0.4.2
+jupyter==1.1.1
+jupyter_nbextensions_configurator==0.6.4
+causal-conv1d==1.5.0.post8
+flash_attn==2.7.4.post1
+mamba-ssm==2.2.4
+hnet==0.0.1
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

routing_rerun/N_2.5/wandb/run-20260523_172819-hcp9axm4/files/wandb-metadata.json

@@ -0,0 +1,44 @@
+{
+  "os": "Linux-6.8.0-117-generic-x86_64-with-glibc2.39",
+  "python": "CPython 3.12.0",
+  "startedAt": "2026-05-23T17:28:19.719235Z",
+  "args": [
+    "training.load_balancing_N=2.5",
+    "tracking.run_name=routing_N2.5"
+  ],
+  "program": "/workspace/byte-llms-code/routing_evolution_exp/train.py",
+  "codePath": "routing_evolution_exp/train.py",
+  "codePathLocal": "train.py",
+  "git": {
+    "remote": "https://github.com/naryst/byte-llms-code.git",
+    "commit": "15d97dd48d00266a0b8f634920ab99cb7547da91"
+  },
+  "email": "nikita@local.ru",
+  "root": "outputs/2026-05-23/N_2.5",
+  "host": "1a023b226280",
+  "executable": "/venv/bytellm/bin/python",
+  "cpu_count": 36,
+  "cpu_count_logical": 72,
+  "gpu": "NVIDIA A100 80GB PCIe",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "234075717632",
+      "used": "19065532416"
+    }
+  },
+  "memory": {
+    "total": "404274495488"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100 80GB PCIe",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ec9764c4-224a-9508-002f-de7762d78498"
+    }
+  ],
+  "cudaVersion": "13.2",
+  "writerId": "bnp9f95qneklhh3kico6ks4t4aw6iad3"
+}

routing_rerun/N_2.5/wandb/run-20260523_172819-hcp9axm4/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":24021,"epoch/soft_boundary_ratio_stage1":0.39492675367323404,"val/perplexity":1.4265099806664625,"epoch/time":8044.9603972435,"_timestamp":1.7795813175301356e+09,"train/epoch":3,"val/loss":0.8547116691308508,"_step":14060,"best/val_perplexity":1.3300726554660505,"train/loss":0.678175687789917,"best/val_loss":0.7847824677543916,"train/soft_boundary_ratio_stage1":0.3949258402291829,"train/chunk_len_stage1":2.4967957973233275,"epoch/loss":0.62571974257963,"epoch/hard_boundary_ratio_stage1":0.4015104655865258,"train/hard_boundary_ratio_stage0":0.4012647159477883,"epoch/lm_loss":0.12546251496211033,"epoch/lb_loss":1.0011933930957395,"train/step_time":1.351996898651123,"train/lb_loss":0.999517560005188,"epoch/hard_boundary_ratio_stage0":0.4012691360589079,"train/hard_boundary_ratio_stage1":0.40151479100268767,"train/loss_avg":0.6257273778828147,"val/time":442.93458580970764,"epoch/chunk_len_stage0":2.4979428233741783,"train/soft_boundary_ratio_stage0":0.43987519941498743,"epoch/soft_boundary_ratio_stage0":0.4398768389750085,"train/lr":1.0000112334056358e-05,"train/chunk_len_stage0":2.497970175538727,"best/step":4000,"epoch/chunk_len_stage1":2.496822725646825,"val/lb_loss":1.0005453147061618,"_wandb":{"runtime":24021},"val/lm_loss":0.3544390111086654,"train/lm_loss":0.18418139219284058}

routing_rerun/N_2.5/wandb/run-20260523_172819-hcp9axm4/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2026-05-23T17:28:19.822583715Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmphin61grg/port-15014.txt","pid":15014,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-05-23T17:28:19.823218456Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":15014}
+{"time":"2026-05-23T17:28:19.823201848Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-15014-15024-2200562283/socket","Net":"unix"}}
+{"time":"2026-05-23T17:28:20.009397582Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-05-23T17:28:20.034058805Z","level":"INFO","msg":"handleInformInit: received","streamId":"hcp9axm4","id":"1(@)"}
+{"time":"2026-05-23T17:28:20.496307068Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"hcp9axm4","id":"1(@)"}
+{"time":"2026-05-24T00:08:43.395067065Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"hcp9axm4","id":"1(@)"}
+{"time":"2026-05-24T00:08:43.401754007Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"hcp9axm4","id":"1(@)"}
+{"time":"2026-05-24T00:08:43.413724477Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-05-24T00:08:43.413783027Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-05-24T00:08:43.413805752Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-05-24T00:08:43.413800194Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-05-24T00:08:43.413876609Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-05-24T00:08:43.413890255Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-05-24T00:08:43.413946146Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-15014-15024-2200562283/socket","Net":"unix"}}
+{"time":"2026-05-24T00:08:43.413995537Z","level":"INFO","msg":"server is closed"}

routing_rerun/N_2.5/wandb/run-20260523_172819-hcp9axm4/logs/debug-internal.log

@@ -0,0 +1,13 @@
+{"time":"2026-05-23T17:28:20.034273503Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-05-23T17:28:20.495979525Z","level":"INFO","msg":"stream: created new stream","id":"hcp9axm4"}
+{"time":"2026-05-23T17:28:20.496106936Z","level":"INFO","msg":"handler: started","stream_id":"hcp9axm4"}
+{"time":"2026-05-23T17:28:20.496281243Z","level":"INFO","msg":"stream: started","id":"hcp9axm4"}
+{"time":"2026-05-23T17:28:20.496287355Z","level":"INFO","msg":"writer: started","stream_id":"hcp9axm4"}
+{"time":"2026-05-23T17:28:20.496309559Z","level":"INFO","msg":"sender: started","stream_id":"hcp9axm4"}
+{"time":"2026-05-23T17:28:20.630740567Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-24T00:08:43.285793193Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-24T00:08:43.389899668Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-24T00:08:43.395126237Z","level":"INFO","msg":"stream: closing","id":"hcp9axm4"}
+{"time":"2026-05-24T00:08:43.395149698Z","level":"INFO","msg":"handler: closed","stream_id":"hcp9axm4"}
+{"time":"2026-05-24T00:08:43.395348295Z","level":"INFO","msg":"sender: closed","stream_id":"hcp9axm4"}
+{"time":"2026-05-24T00:08:43.395364596Z","level":"INFO","msg":"stream: closed","id":"hcp9axm4"}

routing_rerun/N_2.5/wandb/run-20260523_172819-hcp9axm4/logs/debug.log

@@ -0,0 +1,24 @@
+2026-05-23 17:28:19,720 INFO    MainThread:15014 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_setup.py:_flush():81] Configure stats pid to 15014
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/2026-05-23/N_2.5/wandb/run-20260523_172819-hcp9axm4/logs/debug.log
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/2026-05-23/N_2.5/wandb/run-20260523_172819-hcp9axm4/logs/debug-internal.log
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_init.py:init():844] calling init triggers
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 3, 'max_steps': None, 'batch_size': 8, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0001, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.5, 'load_balancing_N': 2.5, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': 150000, 'max_val_samples': None}, 'logging': {'log_interval': 10, 'save_interval': 3000, 'eval_interval': 1000, 'save_every_epoch': False, 'model_only_checkpoints': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'routing-evolution', 'run_name': 'routing_N2.5', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/2026-05-23/N_2.5'}, 'paths': {'output_dir': 'outputs/2026-05-23/N_2.5'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/routing_evolution_exp/train.py'}}
+2026-05-23 17:28:19,721 INFO    MainThread:15014 [wandb_init.py:init():892] starting backend
+2026-05-23 17:28:20,009 INFO    MainThread:15014 [wandb_init.py:init():895] sending inform_init request
+2026-05-23 17:28:20,030 INFO    MainThread:15014 [wandb_init.py:init():903] backend started and connected
+2026-05-23 17:28:20,037 INFO    MainThread:15014 [wandb_init.py:init():973] updated telemetry
+2026-05-23 17:28:20,063 INFO    MainThread:15014 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-23 17:28:20,628 INFO    MainThread:15014 [wandb_init.py:init():1044] starting run threads in backend
+2026-05-23 17:28:20,879 INFO    MainThread:15014 [wandb_run.py:_console_start():2529] atexit reg
+2026-05-23 17:28:20,879 INFO    MainThread:15014 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-05-23 17:28:20,879 INFO    MainThread:15014 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-05-23 17:28:20,879 INFO    MainThread:15014 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-05-23 17:28:20,883 INFO    MainThread:15014 [wandb_init.py:init():1084] run started, returning control to user process
+2026-05-24 00:08:42,341 INFO    MainThread:15014 [wandb_run.py:_finish():2295] finishing run nikita/routing-evolution/hcp9axm4
+2026-05-24 00:08:42,342 INFO    MainThread:15014 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-05-24 00:08:42,342 INFO    MainThread:15014 [wandb_run.py:_restore():2476] restore
+2026-05-24 00:08:42,342 INFO    MainThread:15014 [wandb_run.py:_restore():2482] restore done
+2026-05-24 00:08:43,394 INFO    MainThread:15014 [wandb_run.py:_footer_sync_info():3870] logging synced files

routing_rerun/N_4.0/.hydra/config.yaml

@@ -0,0 +1,57 @@
+model:
+  config_path: ${oc.env:PROJECT_ROOT}/hnet_project/configs/hnet_2stage_XL_code.json
+  checkpoint_path: ${oc.env:PROJECT_ROOT}/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+training:
+  epochs: 3
+  max_steps: null
+  batch_size: 8
+  eval_batch_size: 24
+  gradient_accumulation_steps: 4
+  lr: 0.0001
+  weight_decay: 0.1
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-08
+  lr_scheduler: wsd
+  warmup_ratio: 0.1
+  decay_ratio: 0.2
+  warmup_steps: 100
+  min_lr_ratio: 0.1
+  lr_multiplier:
+  - 2.0
+  - 1.5
+  - 1.0
+  load_balancing_weight: 0.5
+  load_balancing_N: 4.0
+  max_grad_norm: 1.0
+  use_amp: true
+  resume: false
+  resume_checkpoint: null
+  warmup_model: true
+data:
+  path: ${oc.env:PROJECT_ROOT}/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 0
+  pin_memory: true
+  max_train_samples: 150000
+  max_val_samples: null
+logging:
+  log_interval: 10
+  save_interval: 3000
+  eval_interval: 1000
+  save_every_epoch: false
+  model_only_checkpoints: true
+tracking:
+  enabled: true
+  backend: wandb
+  project: routing-evolution
+  run_name: routing_N4.0
+  entity: null
+  base_url: https://wandb.platun0v.ru
+  local_dir: ${paths.output_dir}
+paths:
+  output_dir: outputs/${now:%Y-%m-%d}/N_${training.load_balancing_N}
+seed: 42
+device: cuda

routing_rerun/N_4.0/.hydra/hydra.yaml

@@ -0,0 +1,162 @@
+hydra:
+  run:
+    dir: ${paths.output_dir}
+  sweep:
+    dir: outputs/multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - training.load_balancing_N=4.0
+    - tracking.run_name=routing_N4.0
+  job:
+    name: train
+    chdir: false
+    override_dirname: tracking.run_name=routing_N4.0,training.load_balancing_N=4.0
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace/byte-llms-code/routing_evolution_exp
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/byte-llms-code/routing_evolution_exp/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/byte-llms-code/routing_evolution_exp/outputs/2026-05-24/N_4.0
+    choices:
+      paths: default
+      tracking: default
+      logging: default
+      data: default
+      training: default
+      model: hnet_xl_code
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

routing_rerun/N_4.0/.hydra/overrides.yaml

@@ -0,0 +1,2 @@
+- training.load_balancing_N=4.0
+- tracking.run_name=routing_N4.0

routing_rerun/N_4.0/eval_results/eval_config.yaml

@@ -0,0 +1,29 @@
+model:
+  config_path: /workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json
+  checkpoint_path: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+data:
+  path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 0
+  pin_memory: true
+  max_train_samples: 150000
+  max_val_samples: null
+evaluation:
+  batch_size: 16
+  max_samples: 2000
+  compute_bpb: true
+  bleu_tokenize: none
+  use_amp: true
+  save_predictions: true
+generation:
+  max_length: 256
+  temperature: 0.1
+  top_k: 0
+  top_p: 1.0
+paths:
+  run_dir: outputs/2026-05-24/N_4.0
+  eval_initial: true
+  eval_final: true
+seed: 42
+device: cuda

routing_rerun/N_4.0/eval_results/metrics_model_best.txt

@@ -0,0 +1,17 @@
+Checkpoint: model_best (outputs/2026-05-24/N_4.0/model_best.pt)
+================================================================================
+
+exact_match: 0.343
+token_accuracy: 0.4532350560559354
+bleu: 17.77479978063419
+bpb: 1.3583921951858478
+num_samples: 2000
+gen_wall_time_s: 335.00358938152203
+gen_samples_per_s: 5.970085286824437
+gen_time_per_sample_ms: 167.50179469076102
+gen_chars_per_s: 161.1206617208179
+gen_batch_mean_ms: 2680.0287150521763
+gen_batch_p50_ms: 2478.22286299197
+gen_batch_p95_ms: 5159.001234790774
+gen_batch_max_ms: 7598.14075101167
+gen_num_batches: 125

routing_rerun/N_4.0/eval_results/metrics_model_final.txt

@@ -0,0 +1,17 @@
+Checkpoint: model_final (outputs/2026-05-24/N_4.0/model_final.pt)
+================================================================================
+
+exact_match: 0.3135
+token_accuracy: 0.44612258253396936
+bleu: 16.075966963029884
+bpb: 1.4377865095442746
+num_samples: 2000
+gen_wall_time_s: 352.92306854663184
+gen_samples_per_s: 5.666957414362783
+gen_time_per_sample_ms: 176.46153427331592
+gen_chars_per_s: 162.49433689943845
+gen_batch_mean_ms: 2823.3845483730547
+gen_batch_p50_ms: 2503.8293699617498
+gen_batch_p95_ms: 6290.377801237628
+gen_batch_max_ms: 9379.81953396229
+gen_num_batches: 125

routing_rerun/N_4.0/eval_results/metrics_pretrained.txt

@@ -0,0 +1,17 @@
+Checkpoint: pretrained (/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt)
+================================================================================
+
+exact_match: 0.0
+token_accuracy: 0.46752888904196876
+bleu: 1.0909540935626696
+bpb: 1.8769745278639474
+num_samples: 2000
+gen_wall_time_s: 1711.1607243906474
+gen_samples_per_s: 1.1687972798184751
+gen_time_per_sample_ms: 855.5803621953237
+gen_chars_per_s: 298.4588137866866
+gen_batch_mean_ms: 13689.285795125179
+gen_batch_p50_ms: 13090.146724018268
+gen_batch_p95_ms: 15469.248386588879
+gen_batch_max_ms: 33782.6227840269
+gen_num_batches: 125

routing_rerun/N_4.0/eval_results/summary.txt

@@ -0,0 +1,8 @@
+EVALUATION SUMMARY
+=============================================================================
+
+Checkpoint            Exact Match    Token Acc     BLEU        BPB  ms/sample
+-----------------------------------------------------------------------------
+pretrained                  0.00%       46.75%     1.09     1.8770      855.6
+model_best                 34.30%       45.32%    17.77     1.3584      167.5
+model_final                31.35%       44.61%    16.08     1.4378      176.5

routing_rerun/N_4.0/wandb/debug-internal.log

@@ -0,0 +1,14 @@
+{"time":"2026-05-24T00:08:56.234024021Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-05-24T00:08:56.701391294Z","level":"INFO","msg":"stream: created new stream","id":"j30yot8x"}
+{"time":"2026-05-24T00:08:56.701517641Z","level":"INFO","msg":"handler: started","stream_id":"j30yot8x"}
+{"time":"2026-05-24T00:08:56.701810168Z","level":"INFO","msg":"stream: started","id":"j30yot8x"}
+{"time":"2026-05-24T00:08:56.701847254Z","level":"INFO","msg":"sender: started","stream_id":"j30yot8x"}
+{"time":"2026-05-24T00:08:56.701851256Z","level":"INFO","msg":"writer: started","stream_id":"j30yot8x"}
+{"time":"2026-05-24T00:08:56.8332083Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-05-24T01:16:01.045615831Z","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://wandb.platun0v.ru/files/nikita/routing-evolution/j30yot8x/file_stream","body":"Bad Gateway"}
+{"time":"2026-05-24T05:52:09.949995651Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-05-24T05:52:10.063200014Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-05-24T05:52:10.069795312Z","level":"INFO","msg":"stream: closing","id":"j30yot8x"}
+{"time":"2026-05-24T05:52:10.069837927Z","level":"INFO","msg":"handler: closed","stream_id":"j30yot8x"}
+{"time":"2026-05-24T05:52:10.070129075Z","level":"INFO","msg":"sender: closed","stream_id":"j30yot8x"}
+{"time":"2026-05-24T05:52:10.070155803Z","level":"INFO","msg":"stream: closed","id":"j30yot8x"}

routing_rerun/N_4.0/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_setup.py:_flush():81] Configure stats pid to 72724
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/2026-05-24/N_4.0/wandb/run-20260524_000855-j30yot8x/logs/debug.log
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/2026-05-24/N_4.0/wandb/run-20260524_000855-j30yot8x/logs/debug-internal.log
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_init.py:init():844] calling init triggers
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 3, 'max_steps': None, 'batch_size': 8, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0001, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.5, 'load_balancing_N': 4.0, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': 150000, 'max_val_samples': None}, 'logging': {'log_interval': 10, 'save_interval': 3000, 'eval_interval': 1000, 'save_every_epoch': False, 'model_only_checkpoints': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'routing-evolution', 'run_name': 'routing_N4.0', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/2026-05-24/N_4.0'}, 'paths': {'output_dir': 'outputs/2026-05-24/N_4.0'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/routing_evolution_exp/train.py'}}
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_init.py:init():892] starting backend
+2026-05-24 00:08:56,207 INFO    MainThread:72724 [wandb_init.py:init():895] sending inform_init request
+2026-05-24 00:08:56,229 INFO    MainThread:72724 [wandb_init.py:init():903] backend started and connected
+2026-05-24 00:08:56,235 INFO    MainThread:72724 [wandb_init.py:init():973] updated telemetry
+2026-05-24 00:08:56,260 INFO    MainThread:72724 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-24 00:08:56,831 INFO    MainThread:72724 [wandb_init.py:init():1044] starting run threads in backend
+2026-05-24 00:08:57,079 INFO    MainThread:72724 [wandb_run.py:_console_start():2529] atexit reg
+2026-05-24 00:08:57,079 INFO    MainThread:72724 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-05-24 00:08:57,079 INFO    MainThread:72724 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-05-24 00:08:57,079 INFO    MainThread:72724 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-05-24 00:08:57,085 INFO    MainThread:72724 [wandb_init.py:init():1084] run started, returning control to user process
+2026-05-24 05:52:08,772 INFO    MainThread:72724 [wandb_run.py:_finish():2295] finishing run nikita/routing-evolution/j30yot8x
+2026-05-24 05:52:08,772 INFO    MainThread:72724 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-05-24 05:52:08,772 INFO    MainThread:72724 [wandb_run.py:_restore():2476] restore
+2026-05-24 05:52:08,773 INFO    MainThread:72724 [wandb_run.py:_restore():2482] restore done
+2026-05-24 05:52:10,068 INFO    MainThread:72724 [wandb_run.py:_footer_sync_info():3870] logging synced files

routing_rerun/N_4.0/wandb/run-20260524_000855-j30yot8x/logs/debug.log

@@ -0,0 +1,24 @@
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_setup.py:_flush():81] Configure stats pid to 72724
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/2026-05-24/N_4.0/wandb/run-20260524_000855-j30yot8x/logs/debug.log
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/2026-05-24/N_4.0/wandb/run-20260524_000855-j30yot8x/logs/debug-internal.log
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_init.py:init():844] calling init triggers
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 3, 'max_steps': None, 'batch_size': 8, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0001, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.5, 'load_balancing_N': 4.0, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': 150000, 'max_val_samples': None}, 'logging': {'log_interval': 10, 'save_interval': 3000, 'eval_interval': 1000, 'save_every_epoch': False, 'model_only_checkpoints': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'routing-evolution', 'run_name': 'routing_N4.0', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/2026-05-24/N_4.0'}, 'paths': {'output_dir': 'outputs/2026-05-24/N_4.0'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/routing_evolution_exp/train.py'}}
+2026-05-24 00:08:55,908 INFO    MainThread:72724 [wandb_init.py:init():892] starting backend
+2026-05-24 00:08:56,207 INFO    MainThread:72724 [wandb_init.py:init():895] sending inform_init request
+2026-05-24 00:08:56,229 INFO    MainThread:72724 [wandb_init.py:init():903] backend started and connected
+2026-05-24 00:08:56,235 INFO    MainThread:72724 [wandb_init.py:init():973] updated telemetry
+2026-05-24 00:08:56,260 INFO    MainThread:72724 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-05-24 00:08:56,831 INFO    MainThread:72724 [wandb_init.py:init():1044] starting run threads in backend
+2026-05-24 00:08:57,079 INFO    MainThread:72724 [wandb_run.py:_console_start():2529] atexit reg
+2026-05-24 00:08:57,079 INFO    MainThread:72724 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-05-24 00:08:57,079 INFO    MainThread:72724 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-05-24 00:08:57,079 INFO    MainThread:72724 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-05-24 00:08:57,085 INFO    MainThread:72724 [wandb_init.py:init():1084] run started, returning control to user process
+2026-05-24 05:52:08,772 INFO    MainThread:72724 [wandb_run.py:_finish():2295] finishing run nikita/routing-evolution/j30yot8x
+2026-05-24 05:52:08,772 INFO    MainThread:72724 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-05-24 05:52:08,772 INFO    MainThread:72724 [wandb_run.py:_restore():2476] restore
+2026-05-24 05:52:08,773 INFO    MainThread:72724 [wandb_run.py:_restore():2482] restore done
+2026-05-24 05:52:10,068 INFO    MainThread:72724 [wandb_run.py:_footer_sync_info():3870] logging synced files