Add files using upload-large-folder tool

.gitattributes

@@ -41,3 +41,8 @@ lr_sweep/hnet_xl_code_lr_5e-5/wandb/run-20260425_180603-5xd22ofy/run-5xd22ofy.wa
 lr_sweep/pythia_1b_lr_1e-5/wandb/run-20260425_180609-3z5g26qd/run-3z5g26qd.wandb filter=lfs diff=lfs merge=lfs -text
 lr_sweep/pythia_1b_lr_2e-5/wandb/run-20260425_184822-bhvwo83l/run-bhvwo83l.wandb filter=lfs diff=lfs merge=lfs -text
 lr_sweep/pythia_1b_lr_5e-5/wandb/run-20260425_193045-vg3if73m/run-vg3if73m.wandb filter=lfs diff=lfs merge=lfs -text
+lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/run-i6lt8av0.wandb filter=lfs diff=lfs merge=lfs -text
+lr_sweep/hnet_xl_code_lr_1e-4/wandb/run-20260425_200722-d5usyud5/run-d5usyud5.wandb filter=lfs diff=lfs merge=lfs -text
+lr_sweep/pythia_1b_lr_1e-4/wandb/run-20260425_201333-p8ozhgpm/run-p8ozhgpm.wandb filter=lfs diff=lfs merge=lfs -text
+lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/run-ln6tfunh.wandb filter=lfs diff=lfs merge=lfs -text
+lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/run-khn25dwv.wandb filter=lfs diff=lfs merge=lfs -text

lr_sweep/hnet_xl_code_lr_1e-4/model_final.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5507a5a2456bffdc961775096b62d8f9bebb9dd301fc350b5878213252b348e7
+size 3315165484

lr_sweep/hnet_xl_code_lr_1e-4/wandb/run-20260425_200722-d5usyud5/files/code/code_completion_exp/train_hnet/train.py

@@ -0,0 +1,284 @@
+"""
+Training Pipeline для HNet модели на задаче Code Completion.
+
+Конфигурация через Hydra + OmegaConf, логирование в Trackio.
+Поддержка DDP через Accelerate для multi-GPU тренировки.
+
+Использование:
+    # Базовый запуск (single GPU)
+    python train.py
+
+    # Multi-GPU с Accelerate
+    accelerate launch train.py
+
+    # Multi-GPU с указанием количества GPU
+    accelerate launch --num_processes=4 train.py
+
+    # Переопределение параметров через CLI
+    python train.py training.lr=1e-4 training.epochs=5
+
+    # Выбор другого конфига модели
+    python train.py model=hnet_small
+
+    # Multirun (sweep)
+    python train.py --multirun training.lr=1e-4,3e-4,1e-3
+
+    # Без логирования
+    python train.py tracking.enabled=false
+"""
+
+import os
+import math
+from pathlib import Path
+
+import torch
+import hydra
+from hydra.core.hydra_config import HydraConfig
+from omegaconf import DictConfig, OmegaConf
+from accelerate import Accelerator
+from accelerate.utils import set_seed as accelerate_set_seed
+
+# HNet imports
+from hnet.load_utils import load_from_pretrained, load_from_config
+from hnet.utils.tokenizers import ByteTokenizer
+from hnet.utils.train import group_params
+
+# Ensure repo root is on sys.path (needed when running from subdirectory)
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+# Shared training library
+from training_lib.utils import log_message
+from training_lib.checkpointing import save_checkpoint, load_checkpoint
+from training_lib.schedulers import get_lr_scheduler
+from training_lib.tracking import init_tracking, finish_tracking
+from training_lib.hnet.train_loop import train_epoch
+from training_lib.hnet.data import create_dataloaders
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="config")
+def main(cfg: DictConfig):
+    """Глав��ая функция тренировки с поддержкой DDP чере�� Accelerate."""
+
+    # === Accelerator Setup ===
+    mixed_precision = "bf16" if cfg.training.use_amp else "no"
+
+    accelerator = Accelerator(
+        mixed_precision=mixed_precision,
+        gradient_accumulation_steps=cfg.training.gradient_accumulation_steps,
+    )
+
+    # === Setup ===
+    accelerate_set_seed(cfg.seed)
+
+    if cfg.paths.output_dir is None:
+        cfg.paths.output_dir = HydraConfig.get().runtime.output_dir
+
+    OmegaConf.resolve(cfg)
+
+    log_message(
+        f"CUDA_VISIBLE_DEVICES: {os.environ.get('CUDA_VISIBLE_DEVICES', 'not set')}",
+        cfg,
+        accelerator,
+    )
+    log_message(f"Number of processes: {accelerator.num_processes}", cfg, accelerator)
+    log_message(f"Process index: {accelerator.process_index}", cfg, accelerator)
+    log_message(f"Mixed precision: {mixed_precision}", cfg, accelerator)
+
+    log_message("=" * 60, cfg, accelerator)
+    log_message(
+        "HNet Training Pipeline (Hydra + Trackio + Accelerate)", cfg, accelerator
+    )
+    log_message("=" * 60, cfg, accelerator)
+    log_message(f"Config:\n{OmegaConf.to_yaml(cfg)}", cfg, accelerator)
+
+    # === Trackio Init ===
+    init_tracking(cfg, accelerator)
+
+    # === Tokenizer ===
+    log_message("Initializing tokenizer...", cfg, accelerator)
+    tokenizer = ByteTokenizer()
+
+    # === Model ===
+    log_message("Loading model...", cfg, accelerator)
+    if cfg.model.checkpoint_path:
+        model = load_from_pretrained(
+            model_path=cfg.model.checkpoint_path,
+            model_config_path=cfg.model.config_path,
+        )
+        log_message(f"Loaded pretrained: {cfg.model.checkpoint_path}", cfg, accelerator)
+    else:
+        model = load_from_config(
+            model_config_path=cfg.model.config_path,
+            device="cpu",
+        )
+        model.init_weights()
+        log_message("Initialized from scratch", cfg, accelerator)
+
+    model.train()
+
+    # LR multiplier для разны�� стадий (до prepare!)
+    lr_multiplier = list(cfg.training.lr_multiplier)
+    model.apply_lr_multiplier(lr_multiplier)
+    log_message(f"Applied LR multipliers: {lr_multiplier}", cfg, accelerator)
+
+    # Warmup для Triton kernels
+    if cfg.training.warmup_model:
+        log_message("Warming up model...", cfg, accelerator)
+        model = model.to(accelerator.device)
+        model.warmup(verbose=accelerator.is_main_process)
+
+    # Log model info
+    total_params = sum(p.numel() for p in model.parameters())
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    log_message(f"Total params: {total_params:,}", cfg, accelerator)
+    log_message(f"Trainable params: {trainable_params:,}", cfg, accelerator)
+
+    # === Data ===
+    log_message("Creating dataloaders...", cfg, accelerator)
+    dataloaders = create_dataloaders(cfg, tokenizer)
+
+    train_dataloader = dataloaders["train"]
+    val_dataloader = dataloaders.get("validation", None)
+
+    log_message(
+        f"Train dataset size: {len(train_dataloader.dataset)}", cfg, accelerator
+    )
+    log_message(
+        f"Train batches per epoch (before DDP split): {len(train_dataloader)}",
+        cfg,
+        accelerator,
+    )
+
+    if val_dataloader:
+        log_message(
+            f"Validation dataset size: {len(val_dataloader.dataset)}", cfg, accelerator
+        )
+        log_message(f"Validation batches: {len(val_dataloader)}", cfg, accelerator)
+    else:
+        log_message("No validation dataset found", cfg, accelerator)
+
+    # === Optimizer ===
+    log_message("Creating optimizer...", cfg, accelerator)
+    param_groups = group_params(model)
+
+    for group in param_groups:
+        if "lr" not in group:
+            group["lr"] = cfg.training.lr
+        else:
+            group["lr"] = cfg.training.lr * group.get("lr_multiplier", 1.0)
+        if "weight_decay" not in group:
+            group["weight_decay"] = cfg.training.weight_decay
+
+    optimizer = torch.optim.AdamW(
+        param_groups,
+        lr=cfg.training.lr,
+        betas=tuple(cfg.training.betas),
+        eps=cfg.training.eps,
+    )
+
+    # === Scheduler ===
+    steps_per_epoch = math.ceil(len(train_dataloader) / accelerator.num_processes)
+    total_steps = (
+        cfg.training.epochs
+        * steps_per_epoch
+        // cfg.training.gradient_accumulation_steps
+    )
+    scheduler = get_lr_scheduler(optimizer, cfg, total_steps)
+
+    log_message(
+        f"Total steps: {total_steps}, Steps per epoch: {steps_per_epoch}",
+        cfg,
+        accelerator,
+    )
+
+    # === Accelerate Prepare ===
+    log_message(
+        "Preparing model, optimizer, and dataloaders with Accelerate...",
+        cfg,
+        accelerator,
+    )
+
+    if val_dataloader is not None:
+        model, optimizer, train_dataloader, val_dataloader, scheduler = (
+            accelerator.prepare(
+                model, optimizer, train_dataloader, val_dataloader, scheduler
+            )
+        )
+    else:
+        model, optimizer, train_dataloader, scheduler = accelerator.prepare(
+            model, optimizer, train_dataloader, scheduler
+        )
+
+    log_message(
+        f"Train batches per epoch (after DDP split): {len(train_dataloader)}",
+        cfg,
+        accelerator,
+    )
+
+    # === Resume ===
+    global_step = 0
+    start_epoch = 1
+
+    if cfg.training.resume and cfg.training.resume_checkpoint:
+        global_step, start_epoch = load_checkpoint(
+            model,
+            optimizer,
+            scheduler,
+            cfg.training.resume_checkpoint,
+            cfg,
+            accelerator,
+        )
+        start_epoch += 1
+
+    # === Training Loop ===
+    log_message("Starting training...", cfg, accelerator)
+
+    best_val_loss = float("inf")
+
+    try:
+        for epoch in range(start_epoch, cfg.training.epochs + 1):
+            log_message(f"\n{'=' * 60}", cfg, accelerator)
+            log_message(f"EPOCH {epoch}/{cfg.training.epochs}", cfg, accelerator)
+            log_message(f"{'=' * 60}", cfg, accelerator)
+
+            global_step, best_val_loss = train_epoch(
+                model=model,
+                dataloader=train_dataloader,
+                optimizer=optimizer,
+                scheduler=scheduler,
+                cfg=cfg,
+                epoch=epoch,
+                global_step=global_step,
+                accelerator=accelerator,
+                val_dataloader=val_dataloader,
+                best_val_loss=best_val_loss,
+            )
+
+            if cfg.logging.save_every_epoch:
+                save_checkpoint(
+                    model, optimizer, scheduler, global_step, epoch, cfg, accelerator
+                )
+
+    except KeyboardInterrupt:
+        log_message("Training interrupted by user", cfg, accelerator)
+        save_checkpoint(
+            model, optimizer, scheduler, global_step, epoch, cfg, accelerator
+        )
+
+    # === Final Save ===
+    log_message("\nTraining completed!", cfg, accelerator)
+
+    if accelerator.is_main_process:
+        final_model_path = Path(cfg.paths.output_dir) / "model_final.pt"
+        unwrapped_model = accelerator.unwrap_model(model)
+        torch.save(unwrapped_model.state_dict(), final_model_path)
+        log_message(f"Final model: {final_model_path}", cfg, accelerator)
+
+    accelerator.wait_for_everyone()
+    accelerator.end_training()
+    finish_tracking()
+
+
+if __name__ == "__main__":
+    main()

lr_sweep/hnet_xl_code_lr_1e-4/wandb/run-20260425_200722-d5usyud5/files/config.yaml

@@ -0,0 +1,167 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    code_path: code/code_completion_exp/train_hnet/train.py
+    python_version: 3.12.0
+    cli_version: 0.24.0
+    framework: huggingface
+    huggingface_version: 4.57.6
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1777147642
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 50
+      - 51
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 49
+      - 50
+      - 51
+      - 71
+      - 105
+      3:
+      - 2
+      - 13
+      - 16
+      - 37
+      - 42
+      - 61
+      4: 3.12.0
+      5: 0.24.0
+      6: 4.57.6
+      13: linux-x86_64
+    e:
+      yd4im4gytbm7o9yud168kac4xfyaj2kg:
+        os: Linux-5.4.0-176-generic-x86_64-with-glibc2.35
+        python: CPython 3.12.0
+        started_at: '2026-04-25T20:07:22.253382Z'
+        args:
+        - tracking=wandb
+        - tracking.project=code-completion_lr-sweep
+        - tracking.run_name=hnet_xl_code_lr_1e-4
+        - training.lr=1e-4
+        - paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_1e-4
+        - data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+        program: /workspace/byte-llms-code/code_completion_exp/train_hnet/train.py
+        code_path: code_completion_exp/train_hnet/train.py
+        code_path_local: train.py
+        git:
+          remote_url: https://github.com/naryst/byte-llms-code.git
+          commit: f111e13281aa0dc58e24302edab5b0d5c2024586
+        email: nikita@local.ru
+        root: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_1e-4
+        host: 7504e518d24a
+        executable: /venv/bytellm/bin/python
+        cpu_count: 64
+        cpu_count_logical: 128
+        gpu_type: NVIDIA H100 80GB HBM3
+        gpu_count: 4
+        disk:
+          /:
+            total: '265214230528'
+            used: '104071081984'
+        memory:
+          total: '1081679683584'
+        gpu_nvidia:
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-b60cdcab-2033-2009-41de-be646c953a20
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-9982b420-4520-4238-c378-ec5a46015474
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-e26ebaac-aaa6-3eed-17ab-a3dce303a76f
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-9dfc6dba-0be6-4a10-1027-336cc0e65134
+        cuda_version: '12.2'
+        writer_id: yd4im4gytbm7o9yud168kac4xfyaj2kg
+model:
+  desc: null
+  value:
+    config_path: /workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json
+    checkpoint_path: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+training:
+  desc: null
+  value:
+    epochs: 1
+    batch_size: 4
+    eval_batch_size: 24
+    gradient_accumulation_steps: 4
+    lr: 0.0001
+    weight_decay: 0.1
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    lr_scheduler: wsd
+    warmup_ratio: 0.1
+    decay_ratio: 0.2
+    warmup_steps: 100
+    min_lr_ratio: 0.1
+    lr_multiplier:
+    - 2.0
+    - 1.5
+    - 1.0
+    load_balancing_weight: 0.01
+    load_balancing_N: 4.0
+    max_grad_norm: 1.0
+    use_amp: true
+    resume: false
+    resume_checkpoint: null
+    warmup_model: true
+data:
+  desc: null
+  value:
+    path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+    max_context_len: 4096
+    max_target_len: 256
+    num_workers: 0
+    pin_memory: true
+    max_train_samples: null
+    max_val_samples: 2000
+logging:
+  desc: null
+  value:
+    log_interval: 10
+    save_interval: 0
+    eval_interval: 2000
+    save_every_epoch: false
+tracking:
+  desc: null
+  value:
+    enabled: true
+    backend: wandb
+    project: code-completion_lr-sweep
+    run_name: hnet_xl_code_lr_1e-4
+    entity: null
+    base_url: https://wandb.platun0v.ru
+    local_dir: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_1e-4
+paths:
+  desc: null
+  value:
+    output_dir: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_1e-4
+seed:
+  desc: null
+  value: 42
+device:
+  desc: null
+  value: cuda

lr_sweep/hnet_xl_code_lr_1e-4/wandb/run-20260425_200722-d5usyud5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime": 7158, "train/epoch": 1, "train/step_time": 0.7092468976974488, "train/chunk_len_stage0": 2.8179939558054445, "train/chunk_len_stage1": 2.354251368177087, "_step": 9880, "_timestamp": 1777154798.6939964, "train/loss": 0.19837140291929245, "train/lm_loss": 0.22973915934562683, "train/lb_loss": 1.1289136409759521, "train/hard_boundary_ratio_stage0": 0.35757633658743543, "train/hard_boundary_ratio_stage1": 0.42741741229545455, "train/soft_boundary_ratio_stage1": 0.40072747577085055, "train/loss_avg": 0.29260910248965016, "train/lr": 1e-05, "train/soft_boundary_ratio_stage0": 0.3544859938887565, "val/perplexity": 1.321679014796039, "val/lm_loss": 0.2812648575220789, "val/lb_loss": 1.0910536590076627, "best/step": 8000, "val/time": 4.739992141723633, "best/val_perplexity": 1.321679014796039, "val/loss": 0.29217539443856194, "best/val_loss": 0.29217539443856194, "epoch/chunk_len_stage1": 2.3543435574383285, "epoch/soft_boundary_ratio_stage0": 0.35449292086488965, "epoch/soft_boundary_ratio_stage1": 0.4007127851565031, "epoch/lb_loss": 1.1061990606274803, "epoch/lm_loss": 0.28240673256245047, "epoch/chunk_len_stage0": 2.8179205942465053, "epoch/hard_boundary_ratio_stage0": 0.35758569070574475, "epoch/loss": 0.29261703164552394, "epoch/time": 7105.889644384384, "epoch/hard_boundary_ratio_stage1": 0.42740058972624}

lr_sweep/hnet_xl_code_lr_1e-4/wandb/run-20260425_200722-d5usyud5/run-d5usyud5.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f8b8e6b5350654f7b0216247b78604c928fa1b5d7dab31aa40755bb6c3f5e02
+size 3080041

lr_sweep/hnet_xl_code_lr_2e-4/.hydra/config.yaml

@@ -0,0 +1,55 @@
+model:
+  config_path: ${oc.env:PROJECT_ROOT}/hnet_project/configs/hnet_2stage_XL_code.json
+  checkpoint_path: ${oc.env:PROJECT_ROOT}/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+training:
+  epochs: 1
+  batch_size: 4
+  eval_batch_size: 24
+  gradient_accumulation_steps: 4
+  lr: 0.0002
+  weight_decay: 0.1
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-08
+  lr_scheduler: wsd
+  warmup_ratio: 0.1
+  decay_ratio: 0.2
+  warmup_steps: 100
+  min_lr_ratio: 0.1
+  lr_multiplier:
+  - 2.0
+  - 1.5
+  - 1.0
+  load_balancing_weight: 0.01
+  load_balancing_N: 4.0
+  max_grad_norm: 1.0
+  use_amp: true
+  resume: false
+  resume_checkpoint: null
+  warmup_model: true
+data:
+  path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 0
+  pin_memory: true
+  max_train_samples: null
+  max_val_samples: 2000
+logging:
+  log_interval: 10
+  save_interval: 0
+  eval_interval: 2000
+  save_every_epoch: false
+tracking:
+  enabled: true
+  backend: wandb
+  project: code-completion_lr-sweep
+  run_name: hnet_xl_code_lr_2e-4
+  entity: null
+  base_url: https://wandb.platun0v.ru
+  local_dir: ${paths.output_dir}
+paths:
+  output_dir: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4
+seed: 42
+device: cuda

lr_sweep/hnet_xl_code_lr_2e-4/.hydra/hydra.yaml

@@ -0,0 +1,166 @@
+hydra:
+  run:
+    dir: ${paths.output_dir}
+  sweep:
+    dir: outputs/multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - tracking=wandb
+    - tracking.project=code-completion_lr-sweep
+    - tracking.run_name=hnet_xl_code_lr_2e-4
+    - training.lr=2e-4
+    - paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4
+    - data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+  job:
+    name: train
+    chdir: false
+    override_dirname: data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full,paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4,tracking.project=code-completion_lr-sweep,tracking.run_name=hnet_xl_code_lr_2e-4,tracking=wandb,training.lr=2e-4
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace/byte-llms-code/code_completion_exp/train_hnet
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/byte-llms-code/code_completion_exp/train_hnet/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4
+    choices:
+      paths: default
+      tracking: wandb
+      logging: default
+      data: default
+      training: default
+      model: hnet_xl_code
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

lr_sweep/hnet_xl_code_lr_2e-4/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- tracking=wandb
+- tracking.project=code-completion_lr-sweep
+- tracking.run_name=hnet_xl_code_lr_2e-4
+- training.lr=2e-4
+- paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4
+- data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full

lr_sweep/hnet_xl_code_lr_2e-4/wandb/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2026-04-25T22:20:12.017648642Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-04-25T22:20:12.401332467Z","level":"INFO","msg":"stream: created new stream","id":"khn25dwv"}
+{"time":"2026-04-25T22:20:12.401402455Z","level":"INFO","msg":"handler: started","stream_id":"khn25dwv"}
+{"time":"2026-04-25T22:20:12.401496869Z","level":"INFO","msg":"stream: started","id":"khn25dwv"}
+{"time":"2026-04-25T22:20:12.401507601Z","level":"INFO","msg":"writer: started","stream_id":"khn25dwv"}
+{"time":"2026-04-25T22:20:12.40151694Z","level":"INFO","msg":"sender: started","stream_id":"khn25dwv"}
+{"time":"2026-04-25T22:20:12.5289538Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-04-25T23:20:22.74400376Z","level":"ERROR","msg":"api: HTTP error","status":403,"method":"POST","url":"https://wandb.platun0v.ru/files/nikita/code-completion_lr-sweep/khn25dwv/file_stream"}
+{"time":"2026-04-25T23:20:22.744078123Z","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 403 Forbidden url=https://wandb.platun0v.ru/files/nikita/code-completion_lr-sweep/khn25dwv/file_stream: "}
+{"time":"2026-04-26T00:19:30.370077692Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-26T00:19:30.370863672Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-04-26T00:19:30.373711643Z","level":"INFO","msg":"stream: closing","id":"khn25dwv"}
+{"time":"2026-04-26T00:19:30.373733071Z","level":"INFO","msg":"handler: closed","stream_id":"khn25dwv"}
+{"time":"2026-04-26T00:19:30.37382892Z","level":"INFO","msg":"sender: closed","stream_id":"khn25dwv"}
+{"time":"2026-04-26T00:19:30.373836548Z","level":"INFO","msg":"stream: closed","id":"khn25dwv"}

lr_sweep/hnet_xl_code_lr_2e-4/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_setup.py:_flush():81] Configure stats pid to 198705
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/logs/debug.log
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/logs/debug-internal.log
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_init.py:init():844] calling init triggers
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 1, 'batch_size': 4, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0002, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.01, 'load_balancing_N': 4.0, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': None, 'max_val_samples': 2000}, 'logging': {'log_interval': 10, 'save_interval': 0, 'eval_interval': 2000, 'save_every_epoch': False}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'code-completion_lr-sweep', 'run_name': 'hnet_xl_code_lr_2e-4', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': '/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4'}, 'paths': {'output_dir': '/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/code_completion_exp/train_hnet/train.py'}}
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_init.py:init():892] starting backend
+2026-04-25 22:20:11,992 INFO    MainThread:198705 [wandb_init.py:init():895] sending inform_init request
+2026-04-25 22:20:12,016 INFO    MainThread:198705 [wandb_init.py:init():903] backend started and connected
+2026-04-25 22:20:12,019 INFO    MainThread:198705 [wandb_init.py:init():973] updated telemetry
+2026-04-25 22:20:12,040 INFO    MainThread:198705 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-25 22:20:12,528 INFO    MainThread:198705 [wandb_init.py:init():1044] starting run threads in backend
+2026-04-25 22:20:12,685 INFO    MainThread:198705 [wandb_run.py:_console_start():2529] atexit reg
+2026-04-25 22:20:12,685 INFO    MainThread:198705 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-04-25 22:20:12,685 INFO    MainThread:198705 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-04-25 22:20:12,686 INFO    MainThread:198705 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-04-25 22:20:12,688 INFO    MainThread:198705 [wandb_init.py:init():1084] run started, returning control to user process
+2026-04-26 00:19:29,581 INFO    MainThread:198705 [wandb_run.py:_finish():2295] finishing run nikita/code-completion_lr-sweep/khn25dwv
+2026-04-26 00:19:29,582 INFO    MainThread:198705 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-04-26 00:19:29,582 INFO    MainThread:198705 [wandb_run.py:_restore():2476] restore
+2026-04-26 00:19:29,582 INFO    MainThread:198705 [wandb_run.py:_restore():2482] restore done
+2026-04-26 00:19:30,373 INFO    MainThread:198705 [wandb_run.py:_footer_sync_info():3870] logging synced files

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/files/code/code_completion_exp/train_hnet/train.py

@@ -0,0 +1,284 @@
+"""
+Training Pipeline для HNet модели на задаче Code Completion.
+
+Конфигурация через Hydra + OmegaConf, логирование в Trackio.
+Поддержка DDP через Accelerate для multi-GPU тренировки.
+
+Использование:
+    # Базовый запуск (single GPU)
+    python train.py
+
+    # Multi-GPU с Accelerate
+    accelerate launch train.py
+
+    # Multi-GPU с указанием количества GPU
+    accelerate launch --num_processes=4 train.py
+
+    # Переопределение параметров через CLI
+    python train.py training.lr=1e-4 training.epochs=5
+
+    # Выбор другого конфига модели
+    python train.py model=hnet_small
+
+    # Multirun (sweep)
+    python train.py --multirun training.lr=1e-4,3e-4,1e-3
+
+    # Без логирования
+    python train.py tracking.enabled=false
+"""
+
+import os
+import math
+from pathlib import Path
+
+import torch
+import hydra
+from hydra.core.hydra_config import HydraConfig
+from omegaconf import DictConfig, OmegaConf
+from accelerate import Accelerator
+from accelerate.utils import set_seed as accelerate_set_seed
+
+# HNet imports
+from hnet.load_utils import load_from_pretrained, load_from_config
+from hnet.utils.tokenizers import ByteTokenizer
+from hnet.utils.train import group_params
+
+# Ensure repo root is on sys.path (needed when running from subdirectory)
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+# Shared training library
+from training_lib.utils import log_message
+from training_lib.checkpointing import save_checkpoint, load_checkpoint
+from training_lib.schedulers import get_lr_scheduler
+from training_lib.tracking import init_tracking, finish_tracking
+from training_lib.hnet.train_loop import train_epoch
+from training_lib.hnet.data import create_dataloaders
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="config")
+def main(cfg: DictConfig):
+    """Глав��ая функция тренировки с поддержкой DDP чере�� Accelerate."""
+
+    # === Accelerator Setup ===
+    mixed_precision = "bf16" if cfg.training.use_amp else "no"
+
+    accelerator = Accelerator(
+        mixed_precision=mixed_precision,
+        gradient_accumulation_steps=cfg.training.gradient_accumulation_steps,
+    )
+
+    # === Setup ===
+    accelerate_set_seed(cfg.seed)
+
+    if cfg.paths.output_dir is None:
+        cfg.paths.output_dir = HydraConfig.get().runtime.output_dir
+
+    OmegaConf.resolve(cfg)
+
+    log_message(
+        f"CUDA_VISIBLE_DEVICES: {os.environ.get('CUDA_VISIBLE_DEVICES', 'not set')}",
+        cfg,
+        accelerator,
+    )
+    log_message(f"Number of processes: {accelerator.num_processes}", cfg, accelerator)
+    log_message(f"Process index: {accelerator.process_index}", cfg, accelerator)
+    log_message(f"Mixed precision: {mixed_precision}", cfg, accelerator)
+
+    log_message("=" * 60, cfg, accelerator)
+    log_message(
+        "HNet Training Pipeline (Hydra + Trackio + Accelerate)", cfg, accelerator
+    )
+    log_message("=" * 60, cfg, accelerator)
+    log_message(f"Config:\n{OmegaConf.to_yaml(cfg)}", cfg, accelerator)
+
+    # === Trackio Init ===
+    init_tracking(cfg, accelerator)
+
+    # === Tokenizer ===
+    log_message("Initializing tokenizer...", cfg, accelerator)
+    tokenizer = ByteTokenizer()
+
+    # === Model ===
+    log_message("Loading model...", cfg, accelerator)
+    if cfg.model.checkpoint_path:
+        model = load_from_pretrained(
+            model_path=cfg.model.checkpoint_path,
+            model_config_path=cfg.model.config_path,
+        )
+        log_message(f"Loaded pretrained: {cfg.model.checkpoint_path}", cfg, accelerator)
+    else:
+        model = load_from_config(
+            model_config_path=cfg.model.config_path,
+            device="cpu",
+        )
+        model.init_weights()
+        log_message("Initialized from scratch", cfg, accelerator)
+
+    model.train()
+
+    # LR multiplier для разны�� стадий (до prepare!)
+    lr_multiplier = list(cfg.training.lr_multiplier)
+    model.apply_lr_multiplier(lr_multiplier)
+    log_message(f"Applied LR multipliers: {lr_multiplier}", cfg, accelerator)
+
+    # Warmup для Triton kernels
+    if cfg.training.warmup_model:
+        log_message("Warming up model...", cfg, accelerator)
+        model = model.to(accelerator.device)
+        model.warmup(verbose=accelerator.is_main_process)
+
+    # Log model info
+    total_params = sum(p.numel() for p in model.parameters())
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    log_message(f"Total params: {total_params:,}", cfg, accelerator)
+    log_message(f"Trainable params: {trainable_params:,}", cfg, accelerator)
+
+    # === Data ===
+    log_message("Creating dataloaders...", cfg, accelerator)
+    dataloaders = create_dataloaders(cfg, tokenizer)
+
+    train_dataloader = dataloaders["train"]
+    val_dataloader = dataloaders.get("validation", None)
+
+    log_message(
+        f"Train dataset size: {len(train_dataloader.dataset)}", cfg, accelerator
+    )
+    log_message(
+        f"Train batches per epoch (before DDP split): {len(train_dataloader)}",
+        cfg,
+        accelerator,
+    )
+
+    if val_dataloader:
+        log_message(
+            f"Validation dataset size: {len(val_dataloader.dataset)}", cfg, accelerator
+        )
+        log_message(f"Validation batches: {len(val_dataloader)}", cfg, accelerator)
+    else:
+        log_message("No validation dataset found", cfg, accelerator)
+
+    # === Optimizer ===
+    log_message("Creating optimizer...", cfg, accelerator)
+    param_groups = group_params(model)
+
+    for group in param_groups:
+        if "lr" not in group:
+            group["lr"] = cfg.training.lr
+        else:
+            group["lr"] = cfg.training.lr * group.get("lr_multiplier", 1.0)
+        if "weight_decay" not in group:
+            group["weight_decay"] = cfg.training.weight_decay
+
+    optimizer = torch.optim.AdamW(
+        param_groups,
+        lr=cfg.training.lr,
+        betas=tuple(cfg.training.betas),
+        eps=cfg.training.eps,
+    )
+
+    # === Scheduler ===
+    steps_per_epoch = math.ceil(len(train_dataloader) / accelerator.num_processes)
+    total_steps = (
+        cfg.training.epochs
+        * steps_per_epoch
+        // cfg.training.gradient_accumulation_steps
+    )
+    scheduler = get_lr_scheduler(optimizer, cfg, total_steps)
+
+    log_message(
+        f"Total steps: {total_steps}, Steps per epoch: {steps_per_epoch}",
+        cfg,
+        accelerator,
+    )
+
+    # === Accelerate Prepare ===
+    log_message(
+        "Preparing model, optimizer, and dataloaders with Accelerate...",
+        cfg,
+        accelerator,
+    )
+
+    if val_dataloader is not None:
+        model, optimizer, train_dataloader, val_dataloader, scheduler = (
+            accelerator.prepare(
+                model, optimizer, train_dataloader, val_dataloader, scheduler
+            )
+        )
+    else:
+        model, optimizer, train_dataloader, scheduler = accelerator.prepare(
+            model, optimizer, train_dataloader, scheduler
+        )
+
+    log_message(
+        f"Train batches per epoch (after DDP split): {len(train_dataloader)}",
+        cfg,
+        accelerator,
+    )
+
+    # === Resume ===
+    global_step = 0
+    start_epoch = 1
+
+    if cfg.training.resume and cfg.training.resume_checkpoint:
+        global_step, start_epoch = load_checkpoint(
+            model,
+            optimizer,
+            scheduler,
+            cfg.training.resume_checkpoint,
+            cfg,
+            accelerator,
+        )
+        start_epoch += 1
+
+    # === Training Loop ===
+    log_message("Starting training...", cfg, accelerator)
+
+    best_val_loss = float("inf")
+
+    try:
+        for epoch in range(start_epoch, cfg.training.epochs + 1):
+            log_message(f"\n{'=' * 60}", cfg, accelerator)
+            log_message(f"EPOCH {epoch}/{cfg.training.epochs}", cfg, accelerator)
+            log_message(f"{'=' * 60}", cfg, accelerator)
+
+            global_step, best_val_loss = train_epoch(
+                model=model,
+                dataloader=train_dataloader,
+                optimizer=optimizer,
+                scheduler=scheduler,
+                cfg=cfg,
+                epoch=epoch,
+                global_step=global_step,
+                accelerator=accelerator,
+                val_dataloader=val_dataloader,
+                best_val_loss=best_val_loss,
+            )
+
+            if cfg.logging.save_every_epoch:
+                save_checkpoint(
+                    model, optimizer, scheduler, global_step, epoch, cfg, accelerator
+                )
+
+    except KeyboardInterrupt:
+        log_message("Training interrupted by user", cfg, accelerator)
+        save_checkpoint(
+            model, optimizer, scheduler, global_step, epoch, cfg, accelerator
+        )
+
+    # === Final Save ===
+    log_message("\nTraining completed!", cfg, accelerator)
+
+    if accelerator.is_main_process:
+        final_model_path = Path(cfg.paths.output_dir) / "model_final.pt"
+        unwrapped_model = accelerator.unwrap_model(model)
+        torch.save(unwrapped_model.state_dict(), final_model_path)
+        log_message(f"Final model: {final_model_path}", cfg, accelerator)
+
+    accelerator.wait_for_everyone()
+    accelerator.end_training()
+    finish_tracking()
+
+
+if __name__ == "__main__":
+    main()

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/files/output.log

@@ -0,0 +1,76 @@
+[2026-04-25 22:06:54] Initializing tokenizer...
+[2026-04-25 22:06:54] Loading model...
+[2026-04-25 22:06:58] Loaded pretrained: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+[2026-04-25 22:06:58] Applied LR multipliers: [2.0, 1.5, 1.0]
+[2026-04-25 22:06:58] Warming up model...
+[WARMUP] Starting warmup (compiling Triton kernels)...
+[WARMUP] Forward: 17.363s, Backward: 26.562s
+[WARMUP] Warmup complete. Subsequent passes will be fast.
+[2026-04-25 22:07:42] Total params: 1,654,090,112
+[2026-04-25 22:07:42] Trainable params: 1,654,090,112
+[2026-04-25 22:07:42] Creating dataloaders...
+[2026-04-25 22:07:42] Train dataset size: 316397
+[2026-04-25 22:07:42] Train batches per epoch (before DDP split): 79100
+[2026-04-25 22:07:42] Validation dataset size: 35098
+[2026-04-25 22:07:42] Validation batches: 1463
+[2026-04-25 22:07:42] Creating optimizer...
+[2026-04-25 22:07:42] Total steps: 29662, Steps per epoch: 39550
+[2026-04-25 22:07:42] Preparing model, optimizer, and dataloaders with Accelerate...
+[2026-04-25 22:07:43] Train batches per epoch (after DDP split): 39550
+[2026-04-25 22:07:43] Starting training...
+[2026-04-25 22:07:43]
+============================================================
+[2026-04-25 22:07:43] EPOCH 1/3
+[2026-04-25 22:07:43] ============================================================
+[2026-04-25 22:08:13] Epoch 1 | Step 10 | Loss: 0.6143 | LM: 0.5857 | LB: 1.1576 | CL0: 2.9 | CL1: 2.1 | HR0: 0.352/SR0: 0.351 | HR1: 0.476/SR1: 0.455 | LR: 2.12e-05
+[2026-04-25 22:08:20] Epoch 1 | Step 20 | Loss: 0.5841 | LM: 0.5756 | LB: 1.1555 | CL0: 2.9 | CL1: 2.1 | HR0: 0.352/SR0: 0.351 | HR1: 0.475/SR1: 0.455 | LR: 2.24e-05
+[2026-04-25 22:08:27] Epoch 1 | Step 30 | Loss: 0.5401 | LM: 0.5225 | LB: 1.1531 | CL0: 2.9 | CL1: 2.1 | HR0: 0.352/SR0: 0.351 | HR1: 0.474/SR1: 0.453 | LR: 2.36e-05
+[2026-04-25 22:08:34] Epoch 1 | Step 40 | Loss: 0.5129 | LM: 0.4917 | LB: 1.1617 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.355 | HR1: 0.479/SR1: 0.457 | LR: 2.49e-05
+[2026-04-25 22:08:41] Epoch 1 | Step 50 | Loss: 0.4778 | LM: 0.4483 | LB: 1.1610 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.355 | HR1: 0.477/SR1: 0.457 | LR: 2.61e-05
+[2026-04-25 22:08:48] Epoch 1 | Step 60 | Loss: 0.4532 | LM: 0.4247 | LB: 1.1636 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.355 | HR1: 0.480/SR1: 0.460 | LR: 2.73e-05
+[2026-04-25 22:08:56] Epoch 1 | Step 70 | Loss: 0.4372 | LM: 0.4149 | LB: 1.1636 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.355 | HR1: 0.480/SR1: 0.459 | LR: 2.85e-05
+[2026-04-25 22:09:03] Epoch 1 | Step 80 | Loss: 0.4289 | LM: 0.4151 | LB: 1.1651 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.481/SR1: 0.460 | LR: 2.97e-05
+[2026-04-25 22:09:10] Epoch 1 | Step 90 | Loss: 0.4225 | LM: 0.4074 | LB: 1.1657 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.481/SR1: 0.460 | LR: 3.09e-05
+[2026-04-25 22:09:17] Epoch 1 | Step 100 | Loss: 0.4147 | LM: 0.4106 | LB: 1.1654 | CL0: 2.8 | CL1: 2.1 | HR0: 0.360/SR0: 0.357 | HR1: 0.480/SR1: 0.459 | LR: 3.21e-05
+[2026-04-25 22:09:24] Epoch 1 | Step 110 | Loss: 0.4128 | LM: 0.4158 | LB: 1.1662 | CL0: 2.8 | CL1: 2.1 | HR0: 0.359/SR0: 0.356 | HR1: 0.482/SR1: 0.460 | LR: 3.34e-05
+[2026-04-25 22:09:31] Epoch 1 | Step 120 | Loss: 0.4090 | LM: 0.4045 | LB: 1.1655 | CL0: 2.8 | CL1: 2.1 | HR0: 0.359/SR0: 0.357 | HR1: 0.481/SR1: 0.460 | LR: 3.46e-05
+[2026-04-25 22:09:38] Epoch 1 | Step 130 | Loss: 0.4051 | LM: 0.4000 | LB: 1.1654 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.482/SR1: 0.460 | LR: 3.58e-05
+[2026-04-25 22:09:46] Epoch 1 | Step 140 | Loss: 0.4012 | LM: 0.4007 | LB: 1.1660 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.482/SR1: 0.460 | LR: 3.70e-05
+[2026-04-25 22:09:53] Epoch 1 | Step 150 | Loss: 0.3946 | LM: 0.3875 | LB: 1.1656 | CL0: 2.8 | CL1: 2.1 | HR0: 0.359/SR0: 0.357 | HR1: 0.481/SR1: 0.460 | LR: 3.82e-05
+[2026-04-25 22:10:00] Epoch 1 | Step 160 | Loss: 0.3897 | LM: 0.3822 | LB: 1.1664 | CL0: 2.8 | CL1: 2.1 | HR0: 0.359/SR0: 0.357 | HR1: 0.482/SR1: 0.460 | LR: 3.94e-05
+[2026-04-25 22:10:07] Epoch 1 | Step 170 | Loss: 0.3894 | LM: 0.3884 | LB: 1.1641 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.480/SR1: 0.459 | LR: 4.06e-05
+[2026-04-25 22:10:14] Epoch 1 | Step 180 | Loss: 0.3840 | LM: 0.3834 | LB: 1.1633 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.479/SR1: 0.458 | LR: 4.18e-05
+[2026-04-25 22:10:22] Epoch 1 | Step 190 | Loss: 0.3823 | LM: 0.3808 | LB: 1.1622 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.479/SR1: 0.457 | LR: 4.31e-05
+[2026-04-25 22:10:29] Epoch 1 | Step 200 | Loss: 0.3800 | LM: 0.3834 | LB: 1.1615 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.478/SR1: 0.457 | LR: 4.43e-05
+[2026-04-25 22:10:36] Epoch 1 | Step 210 | Loss: 0.3794 | LM: 0.3856 | LB: 1.1616 | CL0: 2.8 | CL1: 2.1 | HR0: 0.356/SR0: 0.355 | HR1: 0.479/SR1: 0.457 | LR: 4.55e-05
+[2026-04-25 22:10:43] Epoch 1 | Step 220 | Loss: 0.3768 | LM: 0.3858 | LB: 1.1622 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.355 | HR1: 0.479/SR1: 0.458 | LR: 4.67e-05
+[2026-04-25 22:10:50] Epoch 1 | Step 230 | Loss: 0.3727 | LM: 0.3789 | LB: 1.1626 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.355 | HR1: 0.480/SR1: 0.458 | LR: 4.79e-05
+[2026-04-25 22:10:57] Epoch 1 | Step 240 | Loss: 0.3705 | LM: 0.3740 | LB: 1.1623 | CL0: 2.8 | CL1: 2.1 | HR0: 0.356/SR0: 0.355 | HR1: 0.480/SR1: 0.458 | LR: 4.91e-05
+[2026-04-25 22:11:05] Epoch 1 | Step 250 | Loss: 0.3683 | LM: 0.3685 | LB: 1.1624 | CL0: 2.8 | CL1: 2.1 | HR0: 0.356/SR0: 0.355 | HR1: 0.480/SR1: 0.458 | LR: 5.03e-05
+[2026-04-25 22:11:12] Epoch 1 | Step 260 | Loss: 0.3676 | LM: 0.3706 | LB: 1.1626 | CL0: 2.8 | CL1: 2.1 | HR0: 0.356/SR0: 0.354 | HR1: 0.481/SR1: 0.459 | LR: 5.16e-05
+[2026-04-25 22:11:19] Epoch 1 | Step 270 | Loss: 0.3649 | LM: 0.3704 | LB: 1.1618 | CL0: 2.8 | CL1: 2.1 | HR0: 0.356/SR0: 0.354 | HR1: 0.480/SR1: 0.458 | LR: 5.28e-05
+[2026-04-25 22:11:26] Epoch 1 | Step 280 | Loss: 0.3621 | LM: 0.3716 | LB: 1.1620 | CL0: 2.8 | CL1: 2.1 | HR0: 0.356/SR0: 0.354 | HR1: 0.480/SR1: 0.458 | LR: 5.40e-05
+[2026-04-25 22:11:33] Epoch 1 | Step 290 | Loss: 0.3600 | LM: 0.3707 | LB: 1.1622 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.355 | HR1: 0.480/SR1: 0.458 | LR: 5.52e-05
+[2026-04-25 22:11:40] Epoch 1 | Step 300 | Loss: 0.3591 | LM: 0.3701 | LB: 1.1627 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.480/SR1: 0.458 | LR: 5.64e-05
+[2026-04-25 22:11:47] Epoch 1 | Step 310 | Loss: 0.3576 | LM: 0.3678 | LB: 1.1622 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.355 | HR1: 0.480/SR1: 0.457 | LR: 5.76e-05
+[2026-04-25 22:11:55] Epoch 1 | Step 320 | Loss: 0.3549 | LM: 0.3649 | LB: 1.1624 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.480/SR1: 0.457 | LR: 5.88e-05
+[2026-04-25 22:12:02] Epoch 1 | Step 330 | Loss: 0.3530 | LM: 0.3634 | LB: 1.1624 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.480/SR1: 0.457 | LR: 6.01e-05
+[2026-04-25 22:12:09] Epoch 1 | Step 340 | Loss: 0.3509 | LM: 0.3612 | LB: 1.1625 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.480/SR1: 0.457 | LR: 6.13e-05
+[2026-04-25 22:12:16] Epoch 1 | Step 350 | Loss: 0.3499 | LM: 0.3601 | LB: 1.1627 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.480/SR1: 0.458 | LR: 6.25e-05
+[2026-04-25 22:12:23] Epoch 1 | Step 360 | Loss: 0.3473 | LM: 0.3569 | LB: 1.1623 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.480/SR1: 0.457 | LR: 6.37e-05
+[2026-04-25 22:12:30] Epoch 1 | Step 370 | Loss: 0.3450 | LM: 0.3553 | LB: 1.1618 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.479/SR1: 0.457 | LR: 6.49e-05
+[2026-04-25 22:12:37] Epoch 1 | Step 380 | Loss: 0.3431 | LM: 0.3544 | LB: 1.1617 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.479/SR1: 0.457 | LR: 6.61e-05
+[2026-04-25 22:12:45] Epoch 1 | Step 390 | Loss: 0.3420 | LM: 0.3526 | LB: 1.1621 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.479/SR1: 0.457 | LR: 6.73e-05
+[2026-04-25 22:12:52] Epoch 1 | Step 400 | Loss: 0.3414 | LM: 0.3520 | LB: 1.1614 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.479/SR1: 0.456 | LR: 6.86e-05
+[2026-04-25 22:12:59] Epoch 1 | Step 410 | Loss: 0.3401 | LM: 0.3517 | LB: 1.1612 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.479/SR1: 0.456 | LR: 6.98e-05
+[2026-04-25 22:13:06] Epoch 1 | Step 420 | Loss: 0.3388 | LM: 0.3508 | LB: 1.1614 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.479/SR1: 0.456 | LR: 7.10e-05
+[2026-04-25 22:13:13] Epoch 1 | Step 430 | Loss: 0.3385 | LM: 0.3490 | LB: 1.1611 | CL0: 2.8 | CL1: 2.1 | HR0: 0.358/SR0: 0.356 | HR1: 0.479/SR1: 0.456 | LR: 7.22e-05
+[2026-04-25 22:13:20] Epoch 1 | Step 440 | Loss: 0.3366 | LM: 0.3471 | LB: 1.1608 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.355 | HR1: 0.479/SR1: 0.456 | LR: 7.34e-05
+[2026-04-25 22:13:27] Epoch 1 | Step 450 | Loss: 0.3359 | LM: 0.3465 | LB: 1.1605 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.478/SR1: 0.456 | LR: 7.46e-05
+[2026-04-25 22:13:34] Epoch 1 | Step 460 | Loss: 0.3351 | LM: 0.3451 | LB: 1.1606 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.478/SR1: 0.456 | LR: 7.58e-05
+[2026-04-25 22:13:42] Epoch 1 | Step 470 | Loss: 0.3337 | LM: 0.3433 | LB: 1.1603 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.478/SR1: 0.456 | LR: 7.70e-05
+[2026-04-25 22:13:49] Epoch 1 | Step 480 | Loss: 0.3327 | LM: 0.3415 | LB: 1.1603 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.478/SR1: 0.456 | LR: 7.83e-05
+[2026-04-25 22:13:56] Epoch 1 | Step 490 | Loss: 0.3316 | LM: 0.3398 | LB: 1.1603 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.478/SR1: 0.456 | LR: 7.95e-05
+[2026-04-25 22:14:03] Epoch 1 | Step 500 | Loss: 0.3307 | LM: 0.3373 | LB: 1.1600 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.478/SR1: 0.455 | LR: 8.07e-05
+[2026-04-25 22:14:10] Epoch 1 | Step 510 | Loss: 0.3293 | LM: 0.3345 | LB: 1.1598 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.478/SR1: 0.455 | LR: 8.19e-05
+[2026-04-25 22:14:17] Epoch 1 | Step 520 | Loss: 0.3292 | LM: 0.3348 | LB: 1.1594 | CL0: 2.8 | CL1: 2.1 | HR0: 0.357/SR0: 0.356 | HR1: 0.477/SR1: 0.455 | LR: 8.31e-05

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/files/requirements.txt

@@ -0,0 +1,245 @@
+setuptools==78.1.1
+wheel==0.45.1
+pip==25.2
+webencodings==0.5.1
+triton==3.2.0
+pytz==2025.2
+pydub==0.25.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.6.2
+mpmath==1.3.0
+ipython-genutils==0.2.0
+fastjsonschema==2.21.2
+brotli==1.2.0
+antlr4-python3-runtime==4.9.3
+xxhash==3.6.0
+widgetsnbextension==4.0.14
+websocket-client==1.9.0
+webcolors==24.11.1
+wcwidth==0.2.14
+urllib3==2.5.0
+uri-template==1.3.0
+tzdata==2025.2
+typing_extensions==4.15.0
+types-python-dateutil==2.9.0.20251008
+traitlets==5.14.3
+tqdm==4.67.1
+tornado==6.5.2
+tomlkit==0.13.3
+tinycss2==1.4.0
+tabulate==0.9.0
+sympy==1.13.1
+soupsieve==2.8
+sniffio==1.3.1
+smmap==5.0.2
+six==1.17.0
+shellingham==1.5.4
+Send2Trash==1.8.3
+semantic-version==2.10.0
+safetensors==0.6.2
+rpds-py==0.27.1
+rfc3986-validator==0.1.1
+regex==2025.9.18
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-multipart==0.0.22
+python-json-logger==4.0.0
+python-dotenv==1.2.1
+pyparsing==3.2.5
+PyJWT==2.8.0
+Pygments==2.19.2
+pycparser==2.23
+pyarrow==22.0.0
+psutil==7.1.0
+protobuf==6.33.4
+propcache==0.4.1
+prometheus_client==0.23.1
+portalocker==3.2.0
+platformdirs==4.5.0
+pillow==11.3.0
+pexpect==4.9.0
+pathspec==1.0.4
+parso==0.8.5
+pandocfilters==1.5.1
+packaging==25.0
+orjson==3.11.6
+opt_einsum==3.4.0
+nvidia-nvtx-cu12==12.4.127
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nccl-cu12==2.21.5
+nvidia-curand-cu12==10.3.5.147
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cufft-cu12==11.2.1.3
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cublas-cu12==12.4.5.8
+numpy==2.3.3
+ninja==1.13.0
+networkx==3.5
+nest-asyncio==1.6.0
+narwhals==2.15.0
+mypy_extensions==1.1.0
+multidict==6.7.0
+mistune==3.1.4
+mdurl==0.1.2
+MarkupSafe==3.0.3
+lxml==6.0.2
+librt==0.8.0
+lark==1.3.0
+kiwisolver==1.4.9
+jupyterlab_widgets==3.0.15
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.12.1
+itsdangerous==2.2.0
+idna==3.10
+hf-xet==1.1.10
+h11==0.16.0
+groovy==0.1.2
+fsspec==2025.9.0
+frozenlist==1.8.0
+fqdn==1.5.1
+fonttools==4.60.1
+filelock==3.19.1
+ffmpy==1.0.0
+executing==2.2.1
+einops==0.8.1
+dill==0.4.0
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.17
+dacite==1.9.2
+cycler==0.12.1
+comm==0.2.3
+colorama==0.4.6
+click==8.3.1
+charset-normalizer==3.4.3
+certifi==2025.10.5
+bleach==6.2.0
+babel==2.17.0
+attrs==25.4.0
+async-lru==2.0.5
+asttokens==3.0.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+aiofiles==24.1.0
+yarl==1.22.0
+uvicorn==0.40.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+sentry-sdk==2.50.0
+scipy==1.17.0
+sacrebleu==2.6.0
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+reportlab==4.4.9
+referencing==0.36.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+plotly==6.5.2
+pathlib2==2.3.7.post1
+orderedmultidict==1.0.2
+optree==0.17.0
+omegaconf==2.3.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+mypy==1.19.1
+multiprocess==0.70.16
+matplotlib-inline==0.1.7
+markdown-it-py==4.0.0
+jupyter_core==5.8.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+httpcore==1.0.9
+gitdb==4.0.12
+ftfy==6.3.1
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.2
+anyio==4.11.0
+aiosignal==1.4.0
+starlette==0.50.0
+rich==14.2.0
+pydantic==2.12.5
+pandas==2.3.3
+nvidia-cusolver-cu12==11.6.1.9
+matplotlib==3.10.7
+jupyter_server_terminals==0.5.3
+jupyter_client==8.6.3
+jsonschema-specifications==2025.9.1
+ipython==9.6.0
+hydra-core==1.3.2
+huggingface-hub==0.35.3
+httpx==0.28.1
+GitPython==3.1.46
+furl==2.1.4
+cryptography==46.0.4
+arrow==1.3.0
+argon2-cffi-bindings==25.1.0
+aiohttp==3.13.1
+wandb==0.24.0
+typer==0.21.1
+torch==2.6.0
+tokenizers==0.22.1
+seaborn==0.13.2
+safehttpx==0.1.7
+jsonschema==4.25.1
+joypy==0.2.6
+isoduration==20.11.0
+ipywidgets==8.1.7
+ipykernel==6.30.1
+gradio_client==2.0.3
+fastapi==0.128.0
+Authlib==1.6.6
+argon2-cffi==25.1.0
+transformers==4.57.6
+nbformat==5.10.4
+mlstm_kernels==2.0.2
+jupyter-console==6.6.3
+gradio==6.5.1
+datasets==4.3.0
+clearml==1.16.4
+accelerate==1.10.1
+xlstm==2.0.4
+nbclient==0.10.2
+jupyter-events==0.12.0
+trackio==0.15.0
+nbconvert==7.16.6
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.27.3
+jupyter-lsp==2.3.0
+nbclassic==1.3.3
+jupyterlab==4.4.9
+notebook==7.4.7
+jupyter_contrib_core==0.4.2
+jupyter==1.1.1
+jupyter_nbextensions_configurator==0.6.4
+causal-conv1d==1.5.0.post8
+flash_attn==2.7.4.post1
+mamba-ssm==2.2.4
+hnet==0.0.1
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/files/wandb-metadata.json

@@ -0,0 +1,69 @@
+{
+  "os": "Linux-5.4.0-176-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.12.0",
+  "startedAt": "2026-04-25T22:06:53.291858Z",
+  "args": [
+    "tracking=wandb",
+    "tracking.project=code-completion_lr-sweep",
+    "tracking.run_name=hnet_xl_code_lr_2e-4",
+    "training.lr=2e-4",
+    "paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4",
+    "data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full"
+  ],
+  "program": "/workspace/byte-llms-code/code_completion_exp/train_hnet/train.py",
+  "codePath": "code_completion_exp/train_hnet/train.py",
+  "codePathLocal": "train.py",
+  "git": {
+    "remote": "https://github.com/naryst/byte-llms-code.git",
+    "commit": "0a7180b6ab9f63d2794494f09ec4918576d10fa2"
+  },
+  "email": "nikita@local.ru",
+  "root": "/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4",
+  "host": "7504e518d24a",
+  "executable": "/venv/bytellm/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 128,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 4,
+  "disk": {
+    "/": {
+      "total": "265214230528",
+      "used": "121383002112"
+    }
+  },
+  "memory": {
+    "total": "1081679683584"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-b60cdcab-2033-2009-41de-be646c953a20"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-9982b420-4520-4238-c378-ec5a46015474"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-e26ebaac-aaa6-3eed-17ab-a3dce303a76f"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-9dfc6dba-0be6-4a10-1027-336cc0e65134"
+    }
+  ],
+  "cudaVersion": "12.2",
+  "writerId": "pfqfn7olxjo5871ytqpsm7un6nj33lqi"
+}

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2026-04-25T22:06:53.377874817Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp1gnxdtqe/port-191277.txt","pid":191277,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-25T22:06:53.378924529Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":191277}
+{"time":"2026-04-25T22:06:53.378905634Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-191277-191337-3381656369/socket","Net":"unix"}}
+{"time":"2026-04-25T22:06:53.566394193Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-25T22:06:53.589483419Z","level":"INFO","msg":"handleInformInit: received","streamId":"ln6tfunh","id":"1(@)"}
+{"time":"2026-04-25T22:06:53.95731224Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"ln6tfunh","id":"1(@)"}
+{"time":"2026-04-25T22:14:21.261832239Z","level":"INFO","msg":"server: parent process exited, terminating service process"}

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2026-04-25T22:06:53.589583491Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-04-25T22:06:53.957135992Z","level":"INFO","msg":"stream: created new stream","id":"ln6tfunh"}
+{"time":"2026-04-25T22:06:53.957196965Z","level":"INFO","msg":"handler: started","stream_id":"ln6tfunh"}
+{"time":"2026-04-25T22:06:53.957305912Z","level":"INFO","msg":"stream: started","id":"ln6tfunh"}
+{"time":"2026-04-25T22:06:53.95731694Z","level":"INFO","msg":"writer: started","stream_id":"ln6tfunh"}
+{"time":"2026-04-25T22:06:53.957316732Z","level":"INFO","msg":"sender: started","stream_id":"ln6tfunh"}
+{"time":"2026-04-25T22:06:54.080245448Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/logs/debug.log

@@ -0,0 +1,19 @@
+2026-04-25 22:06:53,293 INFO    MainThread:191277 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-04-25 22:06:53,293 INFO    MainThread:191277 [wandb_setup.py:_flush():81] Configure stats pid to 191277
+2026-04-25 22:06:53,293 INFO    MainThread:191277 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-25 22:06:53,293 INFO    MainThread:191277 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/logs/debug.log
+2026-04-25 22:06:53,293 INFO    MainThread:191277 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/logs/debug-internal.log
+2026-04-25 22:06:53,293 INFO    MainThread:191277 [wandb_init.py:init():844] calling init triggers
+2026-04-25 22:06:53,293 INFO    MainThread:191277 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 3, 'batch_size': 4, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0002, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.01, 'load_balancing_N': 4.0, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': None, 'max_val_samples': None}, 'logging': {'log_interval': 10, 'save_interval': 3000, 'eval_interval': 1000, 'save_every_epoch': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'code-completion_lr-sweep', 'run_name': 'hnet_xl_code_lr_2e-4', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': '/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4'}, 'paths': {'output_dir': '/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/code_completion_exp/train_hnet/train.py'}}
+2026-04-25 22:06:53,293 INFO    MainThread:191277 [wandb_init.py:init():892] starting backend
+2026-04-25 22:06:53,566 INFO    MainThread:191277 [wandb_init.py:init():895] sending inform_init request
+2026-04-25 22:06:53,588 INFO    MainThread:191277 [wandb_init.py:init():903] backend started and connected
+2026-04-25 22:06:53,591 INFO    MainThread:191277 [wandb_init.py:init():973] updated telemetry
+2026-04-25 22:06:53,608 INFO    MainThread:191277 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-25 22:06:54,079 INFO    MainThread:191277 [wandb_init.py:init():1044] starting run threads in backend
+2026-04-25 22:06:54,239 INFO    MainThread:191277 [wandb_run.py:_console_start():2529] atexit reg
+2026-04-25 22:06:54,239 INFO    MainThread:191277 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-04-25 22:06:54,239 INFO    MainThread:191277 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-04-25 22:06:54,239 INFO    MainThread:191277 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-04-25 22:06:54,242 INFO    MainThread:191277 [wandb_init.py:init():1084] run started, returning control to user process

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/run-ln6tfunh.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e72032881e7c3bb7ff50124d376cbd2659ebaa1b64f4823985307f64980f864a
+size 163840

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/files/code/code_completion_exp/train_hnet/train.py

@@ -0,0 +1,284 @@
+"""
+Training Pipeline для HNet модели на задаче Code Completion.
+
+Конфигурация через Hydra + OmegaConf, логирование в Trackio.
+Поддержка DDP через Accelerate для multi-GPU тренировки.
+
+Использование:
+    # Базовый запуск (single GPU)
+    python train.py
+
+    # Multi-GPU с Accelerate
+    accelerate launch train.py
+
+    # Multi-GPU с указанием количества GPU
+    accelerate launch --num_processes=4 train.py
+
+    # Переопределение параметров через CLI
+    python train.py training.lr=1e-4 training.epochs=5
+
+    # Выбор другого конфига модели
+    python train.py model=hnet_small
+
+    # Multirun (sweep)
+    python train.py --multirun training.lr=1e-4,3e-4,1e-3
+
+    # Без логирования
+    python train.py tracking.enabled=false
+"""
+
+import os
+import math
+from pathlib import Path
+
+import torch
+import hydra
+from hydra.core.hydra_config import HydraConfig
+from omegaconf import DictConfig, OmegaConf
+from accelerate import Accelerator
+from accelerate.utils import set_seed as accelerate_set_seed
+
+# HNet imports
+from hnet.load_utils import load_from_pretrained, load_from_config
+from hnet.utils.tokenizers import ByteTokenizer
+from hnet.utils.train import group_params
+
+# Ensure repo root is on sys.path (needed when running from subdirectory)
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+# Shared training library
+from training_lib.utils import log_message
+from training_lib.checkpointing import save_checkpoint, load_checkpoint
+from training_lib.schedulers import get_lr_scheduler
+from training_lib.tracking import init_tracking, finish_tracking
+from training_lib.hnet.train_loop import train_epoch
+from training_lib.hnet.data import create_dataloaders
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="config")
+def main(cfg: DictConfig):
+    """Глав��ая функция тренировки с поддержкой DDP чере�� Accelerate."""
+
+    # === Accelerator Setup ===
+    mixed_precision = "bf16" if cfg.training.use_amp else "no"
+
+    accelerator = Accelerator(
+        mixed_precision=mixed_precision,
+        gradient_accumulation_steps=cfg.training.gradient_accumulation_steps,
+    )
+
+    # === Setup ===
+    accelerate_set_seed(cfg.seed)
+
+    if cfg.paths.output_dir is None:
+        cfg.paths.output_dir = HydraConfig.get().runtime.output_dir
+
+    OmegaConf.resolve(cfg)
+
+    log_message(
+        f"CUDA_VISIBLE_DEVICES: {os.environ.get('CUDA_VISIBLE_DEVICES', 'not set')}",
+        cfg,
+        accelerator,
+    )
+    log_message(f"Number of processes: {accelerator.num_processes}", cfg, accelerator)
+    log_message(f"Process index: {accelerator.process_index}", cfg, accelerator)
+    log_message(f"Mixed precision: {mixed_precision}", cfg, accelerator)
+
+    log_message("=" * 60, cfg, accelerator)
+    log_message(
+        "HNet Training Pipeline (Hydra + Trackio + Accelerate)", cfg, accelerator
+    )
+    log_message("=" * 60, cfg, accelerator)
+    log_message(f"Config:\n{OmegaConf.to_yaml(cfg)}", cfg, accelerator)
+
+    # === Trackio Init ===
+    init_tracking(cfg, accelerator)
+
+    # === Tokenizer ===
+    log_message("Initializing tokenizer...", cfg, accelerator)
+    tokenizer = ByteTokenizer()
+
+    # === Model ===
+    log_message("Loading model...", cfg, accelerator)
+    if cfg.model.checkpoint_path:
+        model = load_from_pretrained(
+            model_path=cfg.model.checkpoint_path,
+            model_config_path=cfg.model.config_path,
+        )
+        log_message(f"Loaded pretrained: {cfg.model.checkpoint_path}", cfg, accelerator)
+    else:
+        model = load_from_config(
+            model_config_path=cfg.model.config_path,
+            device="cpu",
+        )
+        model.init_weights()
+        log_message("Initialized from scratch", cfg, accelerator)
+
+    model.train()
+
+    # LR multiplier для разны�� стадий (до prepare!)
+    lr_multiplier = list(cfg.training.lr_multiplier)
+    model.apply_lr_multiplier(lr_multiplier)
+    log_message(f"Applied LR multipliers: {lr_multiplier}", cfg, accelerator)
+
+    # Warmup для Triton kernels
+    if cfg.training.warmup_model:
+        log_message("Warming up model...", cfg, accelerator)
+        model = model.to(accelerator.device)
+        model.warmup(verbose=accelerator.is_main_process)
+
+    # Log model info
+    total_params = sum(p.numel() for p in model.parameters())
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    log_message(f"Total params: {total_params:,}", cfg, accelerator)
+    log_message(f"Trainable params: {trainable_params:,}", cfg, accelerator)
+
+    # === Data ===
+    log_message("Creating dataloaders...", cfg, accelerator)
+    dataloaders = create_dataloaders(cfg, tokenizer)
+
+    train_dataloader = dataloaders["train"]
+    val_dataloader = dataloaders.get("validation", None)
+
+    log_message(
+        f"Train dataset size: {len(train_dataloader.dataset)}", cfg, accelerator
+    )
+    log_message(
+        f"Train batches per epoch (before DDP split): {len(train_dataloader)}",
+        cfg,
+        accelerator,
+    )
+
+    if val_dataloader:
+        log_message(
+            f"Validation dataset size: {len(val_dataloader.dataset)}", cfg, accelerator
+        )
+        log_message(f"Validation batches: {len(val_dataloader)}", cfg, accelerator)
+    else:
+        log_message("No validation dataset found", cfg, accelerator)
+
+    # === Optimizer ===
+    log_message("Creating optimizer...", cfg, accelerator)
+    param_groups = group_params(model)
+
+    for group in param_groups:
+        if "lr" not in group:
+            group["lr"] = cfg.training.lr
+        else:
+            group["lr"] = cfg.training.lr * group.get("lr_multiplier", 1.0)
+        if "weight_decay" not in group:
+            group["weight_decay"] = cfg.training.weight_decay
+
+    optimizer = torch.optim.AdamW(
+        param_groups,
+        lr=cfg.training.lr,
+        betas=tuple(cfg.training.betas),
+        eps=cfg.training.eps,
+    )
+
+    # === Scheduler ===
+    steps_per_epoch = math.ceil(len(train_dataloader) / accelerator.num_processes)
+    total_steps = (
+        cfg.training.epochs
+        * steps_per_epoch
+        // cfg.training.gradient_accumulation_steps
+    )
+    scheduler = get_lr_scheduler(optimizer, cfg, total_steps)
+
+    log_message(
+        f"Total steps: {total_steps}, Steps per epoch: {steps_per_epoch}",
+        cfg,
+        accelerator,
+    )
+
+    # === Accelerate Prepare ===
+    log_message(
+        "Preparing model, optimizer, and dataloaders with Accelerate...",
+        cfg,
+        accelerator,
+    )
+
+    if val_dataloader is not None:
+        model, optimizer, train_dataloader, val_dataloader, scheduler = (
+            accelerator.prepare(
+                model, optimizer, train_dataloader, val_dataloader, scheduler
+            )
+        )
+    else:
+        model, optimizer, train_dataloader, scheduler = accelerator.prepare(
+            model, optimizer, train_dataloader, scheduler
+        )
+
+    log_message(
+        f"Train batches per epoch (after DDP split): {len(train_dataloader)}",
+        cfg,
+        accelerator,
+    )
+
+    # === Resume ===
+    global_step = 0
+    start_epoch = 1
+
+    if cfg.training.resume and cfg.training.resume_checkpoint:
+        global_step, start_epoch = load_checkpoint(
+            model,
+            optimizer,
+            scheduler,
+            cfg.training.resume_checkpoint,
+            cfg,
+            accelerator,
+        )
+        start_epoch += 1
+
+    # === Training Loop ===
+    log_message("Starting training...", cfg, accelerator)
+
+    best_val_loss = float("inf")
+
+    try:
+        for epoch in range(start_epoch, cfg.training.epochs + 1):
+            log_message(f"\n{'=' * 60}", cfg, accelerator)
+            log_message(f"EPOCH {epoch}/{cfg.training.epochs}", cfg, accelerator)
+            log_message(f"{'=' * 60}", cfg, accelerator)
+
+            global_step, best_val_loss = train_epoch(
+                model=model,
+                dataloader=train_dataloader,
+                optimizer=optimizer,
+                scheduler=scheduler,
+                cfg=cfg,
+                epoch=epoch,
+                global_step=global_step,
+                accelerator=accelerator,
+                val_dataloader=val_dataloader,
+                best_val_loss=best_val_loss,
+            )
+
+            if cfg.logging.save_every_epoch:
+                save_checkpoint(
+                    model, optimizer, scheduler, global_step, epoch, cfg, accelerator
+                )
+
+    except KeyboardInterrupt:
+        log_message("Training interrupted by user", cfg, accelerator)
+        save_checkpoint(
+            model, optimizer, scheduler, global_step, epoch, cfg, accelerator
+        )
+
+    # === Final Save ===
+    log_message("\nTraining completed!", cfg, accelerator)
+
+    if accelerator.is_main_process:
+        final_model_path = Path(cfg.paths.output_dir) / "model_final.pt"
+        unwrapped_model = accelerator.unwrap_model(model)
+        torch.save(unwrapped_model.state_dict(), final_model_path)
+        log_message(f"Final model: {final_model_path}", cfg, accelerator)
+
+    accelerator.wait_for_everyone()
+    accelerator.end_training()
+    finish_tracking()
+
+
+if __name__ == "__main__":
+    main()

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/files/config.yaml

@@ -0,0 +1,167 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    code_path: code/code_completion_exp/train_hnet/train.py
+    python_version: 3.12.0
+    cli_version: 0.24.0
+    framework: huggingface
+    huggingface_version: 4.57.6
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1777155612
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 50
+      - 51
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 49
+      - 50
+      - 51
+      - 71
+      - 105
+      3:
+      - 2
+      - 13
+      - 16
+      - 37
+      - 42
+      - 61
+      4: 3.12.0
+      5: 0.24.0
+      6: 4.57.6
+      13: linux-x86_64
+    e:
+      i2jx9zm2jjq81elpzo2fmxkizkbg0bw5:
+        os: Linux-5.4.0-176-generic-x86_64-with-glibc2.35
+        python: CPython 3.12.0
+        started_at: '2026-04-25T22:20:11.717689Z'
+        args:
+        - tracking=wandb
+        - tracking.project=code-completion_lr-sweep
+        - tracking.run_name=hnet_xl_code_lr_2e-4
+        - training.lr=2e-4
+        - paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4
+        - data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+        program: /workspace/byte-llms-code/code_completion_exp/train_hnet/train.py
+        code_path: code_completion_exp/train_hnet/train.py
+        code_path_local: train.py
+        git:
+          remote_url: https://github.com/naryst/byte-llms-code.git
+          commit: 0a7180b6ab9f63d2794494f09ec4918576d10fa2
+        email: nikita@local.ru
+        root: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4
+        host: 7504e518d24a
+        executable: /venv/bytellm/bin/python
+        cpu_count: 64
+        cpu_count_logical: 128
+        gpu_type: NVIDIA H100 80GB HBM3
+        gpu_count: 4
+        disk:
+          /:
+            total: '265214230528'
+            used: '121389543424'
+        memory:
+          total: '1081679683584'
+        gpu_nvidia:
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-b60cdcab-2033-2009-41de-be646c953a20
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-9982b420-4520-4238-c378-ec5a46015474
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-e26ebaac-aaa6-3eed-17ab-a3dce303a76f
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-9dfc6dba-0be6-4a10-1027-336cc0e65134
+        cuda_version: '12.2'
+        writer_id: i2jx9zm2jjq81elpzo2fmxkizkbg0bw5
+model:
+  desc: null
+  value:
+    config_path: /workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json
+    checkpoint_path: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+training:
+  desc: null
+  value:
+    epochs: 1
+    batch_size: 4
+    eval_batch_size: 24
+    gradient_accumulation_steps: 4
+    lr: 0.0002
+    weight_decay: 0.1
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    lr_scheduler: wsd
+    warmup_ratio: 0.1
+    decay_ratio: 0.2
+    warmup_steps: 100
+    min_lr_ratio: 0.1
+    lr_multiplier:
+    - 2.0
+    - 1.5
+    - 1.0
+    load_balancing_weight: 0.01
+    load_balancing_N: 4.0
+    max_grad_norm: 1.0
+    use_amp: true
+    resume: false
+    resume_checkpoint: null
+    warmup_model: true
+data:
+  desc: null
+  value:
+    path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+    max_context_len: 4096
+    max_target_len: 256
+    num_workers: 0
+    pin_memory: true
+    max_train_samples: null
+    max_val_samples: 2000
+logging:
+  desc: null
+  value:
+    log_interval: 10
+    save_interval: 0
+    eval_interval: 2000
+    save_every_epoch: false
+tracking:
+  desc: null
+  value:
+    enabled: true
+    backend: wandb
+    project: code-completion_lr-sweep
+    run_name: hnet_xl_code_lr_2e-4
+    entity: null
+    base_url: https://wandb.platun0v.ru
+    local_dir: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4
+paths:
+  desc: null
+  value:
+    output_dir: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4
+seed:
+  desc: null
+  value: 42
+device:
+  desc: null
+  value: cuda

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/files/requirements.txt

@@ -0,0 +1,245 @@
+setuptools==78.1.1
+wheel==0.45.1
+pip==25.2
+webencodings==0.5.1
+triton==3.2.0
+pytz==2025.2
+pydub==0.25.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.6.2
+mpmath==1.3.0
+ipython-genutils==0.2.0
+fastjsonschema==2.21.2
+brotli==1.2.0
+antlr4-python3-runtime==4.9.3
+xxhash==3.6.0
+widgetsnbextension==4.0.14
+websocket-client==1.9.0
+webcolors==24.11.1
+wcwidth==0.2.14
+urllib3==2.5.0
+uri-template==1.3.0
+tzdata==2025.2
+typing_extensions==4.15.0
+types-python-dateutil==2.9.0.20251008
+traitlets==5.14.3
+tqdm==4.67.1
+tornado==6.5.2
+tomlkit==0.13.3
+tinycss2==1.4.0
+tabulate==0.9.0
+sympy==1.13.1
+soupsieve==2.8
+sniffio==1.3.1
+smmap==5.0.2
+six==1.17.0
+shellingham==1.5.4
+Send2Trash==1.8.3
+semantic-version==2.10.0
+safetensors==0.6.2
+rpds-py==0.27.1
+rfc3986-validator==0.1.1
+regex==2025.9.18
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-multipart==0.0.22
+python-json-logger==4.0.0
+python-dotenv==1.2.1
+pyparsing==3.2.5
+PyJWT==2.8.0
+Pygments==2.19.2
+pycparser==2.23
+pyarrow==22.0.0
+psutil==7.1.0
+protobuf==6.33.4
+propcache==0.4.1
+prometheus_client==0.23.1
+portalocker==3.2.0
+platformdirs==4.5.0
+pillow==11.3.0
+pexpect==4.9.0
+pathspec==1.0.4
+parso==0.8.5
+pandocfilters==1.5.1
+packaging==25.0
+orjson==3.11.6
+opt_einsum==3.4.0
+nvidia-nvtx-cu12==12.4.127
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nccl-cu12==2.21.5
+nvidia-curand-cu12==10.3.5.147
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cufft-cu12==11.2.1.3
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cublas-cu12==12.4.5.8
+numpy==2.3.3
+ninja==1.13.0
+networkx==3.5
+nest-asyncio==1.6.0
+narwhals==2.15.0
+mypy_extensions==1.1.0
+multidict==6.7.0
+mistune==3.1.4
+mdurl==0.1.2
+MarkupSafe==3.0.3
+lxml==6.0.2
+librt==0.8.0
+lark==1.3.0
+kiwisolver==1.4.9
+jupyterlab_widgets==3.0.15
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.12.1
+itsdangerous==2.2.0
+idna==3.10
+hf-xet==1.1.10
+h11==0.16.0
+groovy==0.1.2
+fsspec==2025.9.0
+frozenlist==1.8.0
+fqdn==1.5.1
+fonttools==4.60.1
+filelock==3.19.1
+ffmpy==1.0.0
+executing==2.2.1
+einops==0.8.1
+dill==0.4.0
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.17
+dacite==1.9.2
+cycler==0.12.1
+comm==0.2.3
+colorama==0.4.6
+click==8.3.1
+charset-normalizer==3.4.3
+certifi==2025.10.5
+bleach==6.2.0
+babel==2.17.0
+attrs==25.4.0
+async-lru==2.0.5
+asttokens==3.0.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+aiofiles==24.1.0
+yarl==1.22.0
+uvicorn==0.40.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+sentry-sdk==2.50.0
+scipy==1.17.0
+sacrebleu==2.6.0
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+reportlab==4.4.9
+referencing==0.36.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+plotly==6.5.2
+pathlib2==2.3.7.post1
+orderedmultidict==1.0.2
+optree==0.17.0
+omegaconf==2.3.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+mypy==1.19.1
+multiprocess==0.70.16
+matplotlib-inline==0.1.7
+markdown-it-py==4.0.0
+jupyter_core==5.8.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+httpcore==1.0.9
+gitdb==4.0.12
+ftfy==6.3.1
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.2
+anyio==4.11.0
+aiosignal==1.4.0
+starlette==0.50.0
+rich==14.2.0
+pydantic==2.12.5
+pandas==2.3.3
+nvidia-cusolver-cu12==11.6.1.9
+matplotlib==3.10.7
+jupyter_server_terminals==0.5.3
+jupyter_client==8.6.3
+jsonschema-specifications==2025.9.1
+ipython==9.6.0
+hydra-core==1.3.2
+huggingface-hub==0.35.3
+httpx==0.28.1
+GitPython==3.1.46
+furl==2.1.4
+cryptography==46.0.4
+arrow==1.3.0
+argon2-cffi-bindings==25.1.0
+aiohttp==3.13.1
+wandb==0.24.0
+typer==0.21.1
+torch==2.6.0
+tokenizers==0.22.1
+seaborn==0.13.2
+safehttpx==0.1.7
+jsonschema==4.25.1
+joypy==0.2.6
+isoduration==20.11.0
+ipywidgets==8.1.7
+ipykernel==6.30.1
+gradio_client==2.0.3
+fastapi==0.128.0
+Authlib==1.6.6
+argon2-cffi==25.1.0
+transformers==4.57.6
+nbformat==5.10.4
+mlstm_kernels==2.0.2
+jupyter-console==6.6.3
+gradio==6.5.1
+datasets==4.3.0
+clearml==1.16.4
+accelerate==1.10.1
+xlstm==2.0.4
+nbclient==0.10.2
+jupyter-events==0.12.0
+trackio==0.15.0
+nbconvert==7.16.6
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.27.3
+jupyter-lsp==2.3.0
+nbclassic==1.3.3
+jupyterlab==4.4.9
+notebook==7.4.7
+jupyter_contrib_core==0.4.2
+jupyter==1.1.1
+jupyter_nbextensions_configurator==0.6.4
+causal-conv1d==1.5.0.post8
+flash_attn==2.7.4.post1
+mamba-ssm==2.2.4
+hnet==0.0.1
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/files/wandb-metadata.json

@@ -0,0 +1 @@
+{"os": "Linux-5.4.0-176-generic-x86_64-with-glibc2.35", "python": "CPython 3.12.0", "started_at": "2026-04-25T22:20:11.717689Z", "args": ["tracking=wandb", "tracking.project=code-completion_lr-sweep", "tracking.run_name=hnet_xl_code_lr_2e-4", "training.lr=2e-4", "paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4", "data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full"], "program": "/workspace/byte-llms-code/code_completion_exp/train_hnet/train.py", "code_path": "code_completion_exp/train_hnet/train.py", "code_path_local": "train.py", "git": {"remote_url": "https://github.com/naryst/byte-llms-code.git", "commit": "0a7180b6ab9f63d2794494f09ec4918576d10fa2"}, "email": "nikita@local.ru", "root": "/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4", "host": "7504e518d24a", "executable": "/venv/bytellm/bin/python", "cpu_count": 64, "cpu_count_logical": 128, "gpu_type": "NVIDIA H100 80GB HBM3", "gpu_count": 4, "disk": {"/": {"total": "265214230528", "used": "121389543424"}}, "memory": {"total": "1081679683584"}, "gpu_nvidia": [{"name": "NVIDIA H100 80GB HBM3", "memory_total": "85520809984", "cuda_cores": 16896, "architecture": "Hopper", "uuid": "GPU-b60cdcab-2033-2009-41de-be646c953a20"}, {"name": "NVIDIA H100 80GB HBM3", "memory_total": "85520809984", "cuda_cores": 16896, "architecture": "Hopper", "uuid": "GPU-9982b420-4520-4238-c378-ec5a46015474"}, {"name": "NVIDIA H100 80GB HBM3", "memory_total": "85520809984", "cuda_cores": 16896, "architecture": "Hopper", "uuid": "GPU-e26ebaac-aaa6-3eed-17ab-a3dce303a76f"}, {"name": "NVIDIA H100 80GB HBM3", "memory_total": "85520809984", "cuda_cores": 16896, "architecture": "Hopper", "uuid": "GPU-9dfc6dba-0be6-4a10-1027-336cc0e65134"}], "cuda_version": "12.2", "writer_id": "i2jx9zm2jjq81elpzo2fmxkizkbg0bw5"}

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime": 7157, "train/loss_avg": 0.30111300496024884, "train/lm_loss": 0.22366735339164734, "train/step_time": 0.7080746650695801, "train/hard_boundary_ratio_stage0": 0.35731984922330645, "train/soft_boundary_ratio_stage1": 0.3884844363677051, "_step": 9880, "train/lb_loss": 1.1152780055999756, "train/lr": 2e-05, "train/chunk_len_stage1": 2.422819767367862, "train/loss": 0.19969405978918076, "train/epoch": 1, "train/chunk_len_stage0": 2.819989316907469, "train/hard_boundary_ratio_stage1": 0.41531218191156627, "train/soft_boundary_ratio_stage0": 0.35144078876136436, "_timestamp": 1777162766.8844316, "val/perplexity": 1.3305252412493118, "val/lb_loss": 1.0830148203032357, "val/lm_loss": 0.2882710979098365, "val/time": 4.704254865646362, "best/step": 8000, "val/loss": 0.29910124660957427, "best/val_loss": 0.29910124660957427, "best/val_perplexity": 1.3305252412493118, "epoch/time": 7104.988562345505, "epoch/soft_boundary_ratio_stage0": 0.3514468830948854, "epoch/loss": 0.30111785081837467, "epoch/hard_boundary_ratio_stage0": 0.3573285990023209, "epoch/lm_loss": 0.2911626837279241, "epoch/lb_loss": 1.0948496247683472, "epoch/chunk_len_stage0": 2.8199204649604765, "epoch/chunk_len_stage1": 2.4228836114764665, "epoch/hard_boundary_ratio_stage1": 0.41530060060195706, "epoch/soft_boundary_ratio_stage1": 0.38847202075385673}

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2026-04-25T22:20:11.804586482Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpq3353tb5/port-198705.txt","pid":198705,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-25T22:20:11.804980932Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":198705}
+{"time":"2026-04-25T22:20:11.804989464Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-198705-198765-1964719290/socket","Net":"unix"}}
+{"time":"2026-04-25T22:20:11.99240949Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-25T22:20:12.017529004Z","level":"INFO","msg":"handleInformInit: received","streamId":"khn25dwv","id":"1(@)"}
+{"time":"2026-04-25T22:20:12.401503529Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"khn25dwv","id":"1(@)"}
+{"time":"2026-04-26T00:19:30.37366386Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"khn25dwv","id":"1(@)"}
+{"time":"2026-04-26T00:19:30.37416302Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"khn25dwv","id":"1(@)"}
+{"time":"2026-04-26T00:19:30.385004444Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-26T00:19:30.385029606Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-26T00:19:30.38503557Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-26T00:19:30.385040394Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-26T00:19:30.385092793Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-26T00:19:30.385102377Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-26T00:19:30.385084287Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-198705-198765-1964719290/socket","Net":"unix"}}
+{"time":"2026-04-26T00:19:30.385111472Z","level":"INFO","msg":"server is closed"}

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2026-04-25T22:20:12.017648642Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-04-25T22:20:12.401332467Z","level":"INFO","msg":"stream: created new stream","id":"khn25dwv"}
+{"time":"2026-04-25T22:20:12.401402455Z","level":"INFO","msg":"handler: started","stream_id":"khn25dwv"}
+{"time":"2026-04-25T22:20:12.401496869Z","level":"INFO","msg":"stream: started","id":"khn25dwv"}
+{"time":"2026-04-25T22:20:12.401507601Z","level":"INFO","msg":"writer: started","stream_id":"khn25dwv"}
+{"time":"2026-04-25T22:20:12.40151694Z","level":"INFO","msg":"sender: started","stream_id":"khn25dwv"}
+{"time":"2026-04-25T22:20:12.5289538Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-04-25T23:20:22.74400376Z","level":"ERROR","msg":"api: HTTP error","status":403,"method":"POST","url":"https://wandb.platun0v.ru/files/nikita/code-completion_lr-sweep/khn25dwv/file_stream"}
+{"time":"2026-04-25T23:20:22.744078123Z","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 403 Forbidden url=https://wandb.platun0v.ru/files/nikita/code-completion_lr-sweep/khn25dwv/file_stream: "}
+{"time":"2026-04-26T00:19:30.370077692Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-26T00:19:30.370863672Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-04-26T00:19:30.373711643Z","level":"INFO","msg":"stream: closing","id":"khn25dwv"}
+{"time":"2026-04-26T00:19:30.373733071Z","level":"INFO","msg":"handler: closed","stream_id":"khn25dwv"}
+{"time":"2026-04-26T00:19:30.37382892Z","level":"INFO","msg":"sender: closed","stream_id":"khn25dwv"}
+{"time":"2026-04-26T00:19:30.373836548Z","level":"INFO","msg":"stream: closed","id":"khn25dwv"}

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/logs/debug.log

@@ -0,0 +1,24 @@
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_setup.py:_flush():81] Configure stats pid to 198705
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/logs/debug.log
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/logs/debug-internal.log
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_init.py:init():844] calling init triggers
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 1, 'batch_size': 4, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0002, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.01, 'load_balancing_N': 4.0, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': None, 'max_val_samples': 2000}, 'logging': {'log_interval': 10, 'save_interval': 0, 'eval_interval': 2000, 'save_every_epoch': False}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'code-completion_lr-sweep', 'run_name': 'hnet_xl_code_lr_2e-4', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': '/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4'}, 'paths': {'output_dir': '/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_2e-4'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/code_completion_exp/train_hnet/train.py'}}
+2026-04-25 22:20:11,719 INFO    MainThread:198705 [wandb_init.py:init():892] starting backend
+2026-04-25 22:20:11,992 INFO    MainThread:198705 [wandb_init.py:init():895] sending inform_init request
+2026-04-25 22:20:12,016 INFO    MainThread:198705 [wandb_init.py:init():903] backend started and connected
+2026-04-25 22:20:12,019 INFO    MainThread:198705 [wandb_init.py:init():973] updated telemetry
+2026-04-25 22:20:12,040 INFO    MainThread:198705 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-25 22:20:12,528 INFO    MainThread:198705 [wandb_init.py:init():1044] starting run threads in backend
+2026-04-25 22:20:12,685 INFO    MainThread:198705 [wandb_run.py:_console_start():2529] atexit reg
+2026-04-25 22:20:12,685 INFO    MainThread:198705 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-04-25 22:20:12,685 INFO    MainThread:198705 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-04-25 22:20:12,686 INFO    MainThread:198705 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-04-25 22:20:12,688 INFO    MainThread:198705 [wandb_init.py:init():1084] run started, returning control to user process
+2026-04-26 00:19:29,581 INFO    MainThread:198705 [wandb_run.py:_finish():2295] finishing run nikita/code-completion_lr-sweep/khn25dwv
+2026-04-26 00:19:29,582 INFO    MainThread:198705 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-04-26 00:19:29,582 INFO    MainThread:198705 [wandb_run.py:_restore():2476] restore
+2026-04-26 00:19:29,582 INFO    MainThread:198705 [wandb_run.py:_restore():2482] restore done
+2026-04-26 00:19:30,373 INFO    MainThread:198705 [wandb_run.py:_footer_sync_info():3870] logging synced files

lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/run-khn25dwv.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b17f5b1390172e14c28b32a79085198d72420d310c8a5d213678c06b1e2749a4
+size 3073785

lr_sweep/hnet_xl_code_lr_5e-4/.hydra/config.yaml

@@ -0,0 +1,55 @@
+model:
+  config_path: ${oc.env:PROJECT_ROOT}/hnet_project/configs/hnet_2stage_XL_code.json
+  checkpoint_path: ${oc.env:PROJECT_ROOT}/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+training:
+  epochs: 1
+  batch_size: 4
+  eval_batch_size: 24
+  gradient_accumulation_steps: 4
+  lr: 0.0005
+  weight_decay: 0.1
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-08
+  lr_scheduler: wsd
+  warmup_ratio: 0.1
+  decay_ratio: 0.2
+  warmup_steps: 100
+  min_lr_ratio: 0.1
+  lr_multiplier:
+  - 2.0
+  - 1.5
+  - 1.0
+  load_balancing_weight: 0.01
+  load_balancing_N: 4.0
+  max_grad_norm: 1.0
+  use_amp: true
+  resume: false
+  resume_checkpoint: null
+  warmup_model: true
+data:
+  path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 0
+  pin_memory: true
+  max_train_samples: null
+  max_val_samples: 2000
+logging:
+  log_interval: 10
+  save_interval: 0
+  eval_interval: 2000
+  save_every_epoch: false
+tracking:
+  enabled: true
+  backend: wandb
+  project: code-completion_lr-sweep
+  run_name: hnet_xl_code_lr_5e-4
+  entity: null
+  base_url: https://wandb.platun0v.ru
+  local_dir: ${paths.output_dir}
+paths:
+  output_dir: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4
+seed: 42
+device: cuda

lr_sweep/hnet_xl_code_lr_5e-4/.hydra/hydra.yaml

@@ -0,0 +1,166 @@
+hydra:
+  run:
+    dir: ${paths.output_dir}
+  sweep:
+    dir: outputs/multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - tracking=wandb
+    - tracking.project=code-completion_lr-sweep
+    - tracking.run_name=hnet_xl_code_lr_5e-4
+    - training.lr=5e-4
+    - paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4
+    - data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+  job:
+    name: train
+    chdir: false
+    override_dirname: data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full,paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4,tracking.project=code-completion_lr-sweep,tracking.run_name=hnet_xl_code_lr_5e-4,tracking=wandb,training.lr=5e-4
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace/byte-llms-code/code_completion_exp/train_hnet
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/byte-llms-code/code_completion_exp/train_hnet/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4
+    choices:
+      paths: default
+      tracking: wandb
+      logging: default
+      data: default
+      training: default
+      model: hnet_xl_code
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

lr_sweep/hnet_xl_code_lr_5e-4/.hydra/overrides.yaml

@@ -0,0 +1,6 @@
+- tracking=wandb
+- tracking.project=code-completion_lr-sweep
+- tracking.run_name=hnet_xl_code_lr_5e-4
+- training.lr=5e-4
+- paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4
+- data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full

lr_sweep/hnet_xl_code_lr_5e-4/model_final.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ca40ecd3344a7b1e8dd106b2e920a9fd77225fbebe1fc4969d809db4074cf77
+size 3315165484

lr_sweep/hnet_xl_code_lr_5e-4/wandb/debug-internal.log

@@ -0,0 +1,13 @@
+{"time":"2026-04-26T00:19:41.489729754Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-04-26T00:19:41.862602753Z","level":"INFO","msg":"stream: created new stream","id":"i6lt8av0"}
+{"time":"2026-04-26T00:19:41.862649982Z","level":"INFO","msg":"handler: started","stream_id":"i6lt8av0"}
+{"time":"2026-04-26T00:19:41.862747075Z","level":"INFO","msg":"stream: started","id":"i6lt8av0"}
+{"time":"2026-04-26T00:19:41.862757167Z","level":"INFO","msg":"writer: started","stream_id":"i6lt8av0"}
+{"time":"2026-04-26T00:19:41.862759475Z","level":"INFO","msg":"sender: started","stream_id":"i6lt8av0"}
+{"time":"2026-04-26T00:19:41.985346468Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-04-26T02:18:38.932345179Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-26T02:18:39.035750391Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-04-26T02:18:39.038119733Z","level":"INFO","msg":"stream: closing","id":"i6lt8av0"}
+{"time":"2026-04-26T02:18:39.038132189Z","level":"INFO","msg":"handler: closed","stream_id":"i6lt8av0"}
+{"time":"2026-04-26T02:18:39.038193257Z","level":"INFO","msg":"sender: closed","stream_id":"i6lt8av0"}
+{"time":"2026-04-26T02:18:39.038198898Z","level":"INFO","msg":"stream: closed","id":"i6lt8av0"}

lr_sweep/hnet_xl_code_lr_5e-4/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_setup.py:_flush():81] Configure stats pid to 257611
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/logs/debug.log
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/logs/debug-internal.log
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_init.py:init():844] calling init triggers
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 1, 'batch_size': 4, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0005, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.01, 'load_balancing_N': 4.0, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': None, 'max_val_samples': 2000}, 'logging': {'log_interval': 10, 'save_interval': 0, 'eval_interval': 2000, 'save_every_epoch': False}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'code-completion_lr-sweep', 'run_name': 'hnet_xl_code_lr_5e-4', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': '/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4'}, 'paths': {'output_dir': '/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/code_completion_exp/train_hnet/train.py'}}
+2026-04-26 00:19:41,199 INFO    MainThread:257611 [wandb_init.py:init():892] starting backend
+2026-04-26 00:19:41,468 INFO    MainThread:257611 [wandb_init.py:init():895] sending inform_init request
+2026-04-26 00:19:41,488 INFO    MainThread:257611 [wandb_init.py:init():903] backend started and connected
+2026-04-26 00:19:41,491 INFO    MainThread:257611 [wandb_init.py:init():973] updated telemetry
+2026-04-26 00:19:41,508 INFO    MainThread:257611 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-26 00:19:41,984 INFO    MainThread:257611 [wandb_init.py:init():1044] starting run threads in backend
+2026-04-26 00:19:42,142 INFO    MainThread:257611 [wandb_run.py:_console_start():2529] atexit reg
+2026-04-26 00:19:42,142 INFO    MainThread:257611 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-04-26 00:19:42,142 INFO    MainThread:257611 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-04-26 00:19:42,142 INFO    MainThread:257611 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-04-26 00:19:42,145 INFO    MainThread:257611 [wandb_init.py:init():1084] run started, returning control to user process
+2026-04-26 02:18:37,753 INFO    MainThread:257611 [wandb_run.py:_finish():2295] finishing run nikita/code-completion_lr-sweep/i6lt8av0
+2026-04-26 02:18:37,754 INFO    MainThread:257611 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-04-26 02:18:37,754 INFO    MainThread:257611 [wandb_run.py:_restore():2476] restore
+2026-04-26 02:18:37,754 INFO    MainThread:257611 [wandb_run.py:_restore():2482] restore done
+2026-04-26 02:18:39,037 INFO    MainThread:257611 [wandb_run.py:_footer_sync_info():3870] logging synced files

lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/files/code/code_completion_exp/train_hnet/train.py

@@ -0,0 +1,284 @@
+"""
+Training Pipeline для HNet модели на задаче Code Completion.
+
+Конфигурация через Hydra + OmegaConf, логирование в Trackio.
+Поддержка DDP через Accelerate для multi-GPU тренировки.
+
+Использование:
+    # Базовый запуск (single GPU)
+    python train.py
+
+    # Multi-GPU с Accelerate
+    accelerate launch train.py
+
+    # Multi-GPU с указанием количества GPU
+    accelerate launch --num_processes=4 train.py
+
+    # Переопределение параметров через CLI
+    python train.py training.lr=1e-4 training.epochs=5
+
+    # Выбор другого конфига модели
+    python train.py model=hnet_small
+
+    # Multirun (sweep)
+    python train.py --multirun training.lr=1e-4,3e-4,1e-3
+
+    # Без логирования
+    python train.py tracking.enabled=false
+"""
+
+import os
+import math
+from pathlib import Path
+
+import torch
+import hydra
+from hydra.core.hydra_config import HydraConfig
+from omegaconf import DictConfig, OmegaConf
+from accelerate import Accelerator
+from accelerate.utils import set_seed as accelerate_set_seed
+
+# HNet imports
+from hnet.load_utils import load_from_pretrained, load_from_config
+from hnet.utils.tokenizers import ByteTokenizer
+from hnet.utils.train import group_params
+
+# Ensure repo root is on sys.path (needed when running from subdirectory)
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+# Shared training library
+from training_lib.utils import log_message
+from training_lib.checkpointing import save_checkpoint, load_checkpoint
+from training_lib.schedulers import get_lr_scheduler
+from training_lib.tracking import init_tracking, finish_tracking
+from training_lib.hnet.train_loop import train_epoch
+from training_lib.hnet.data import create_dataloaders
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="config")
+def main(cfg: DictConfig):
+    """Глав��ая функция тренировки с поддержкой DDP чере�� Accelerate."""
+
+    # === Accelerator Setup ===
+    mixed_precision = "bf16" if cfg.training.use_amp else "no"
+
+    accelerator = Accelerator(
+        mixed_precision=mixed_precision,
+        gradient_accumulation_steps=cfg.training.gradient_accumulation_steps,
+    )
+
+    # === Setup ===
+    accelerate_set_seed(cfg.seed)
+
+    if cfg.paths.output_dir is None:
+        cfg.paths.output_dir = HydraConfig.get().runtime.output_dir
+
+    OmegaConf.resolve(cfg)
+
+    log_message(
+        f"CUDA_VISIBLE_DEVICES: {os.environ.get('CUDA_VISIBLE_DEVICES', 'not set')}",
+        cfg,
+        accelerator,
+    )
+    log_message(f"Number of processes: {accelerator.num_processes}", cfg, accelerator)
+    log_message(f"Process index: {accelerator.process_index}", cfg, accelerator)
+    log_message(f"Mixed precision: {mixed_precision}", cfg, accelerator)
+
+    log_message("=" * 60, cfg, accelerator)
+    log_message(
+        "HNet Training Pipeline (Hydra + Trackio + Accelerate)", cfg, accelerator
+    )
+    log_message("=" * 60, cfg, accelerator)
+    log_message(f"Config:\n{OmegaConf.to_yaml(cfg)}", cfg, accelerator)
+
+    # === Trackio Init ===
+    init_tracking(cfg, accelerator)
+
+    # === Tokenizer ===
+    log_message("Initializing tokenizer...", cfg, accelerator)
+    tokenizer = ByteTokenizer()
+
+    # === Model ===
+    log_message("Loading model...", cfg, accelerator)
+    if cfg.model.checkpoint_path:
+        model = load_from_pretrained(
+            model_path=cfg.model.checkpoint_path,
+            model_config_path=cfg.model.config_path,
+        )
+        log_message(f"Loaded pretrained: {cfg.model.checkpoint_path}", cfg, accelerator)
+    else:
+        model = load_from_config(
+            model_config_path=cfg.model.config_path,
+            device="cpu",
+        )
+        model.init_weights()
+        log_message("Initialized from scratch", cfg, accelerator)
+
+    model.train()
+
+    # LR multiplier для разны�� стадий (до prepare!)
+    lr_multiplier = list(cfg.training.lr_multiplier)
+    model.apply_lr_multiplier(lr_multiplier)
+    log_message(f"Applied LR multipliers: {lr_multiplier}", cfg, accelerator)
+
+    # Warmup для Triton kernels
+    if cfg.training.warmup_model:
+        log_message("Warming up model...", cfg, accelerator)
+        model = model.to(accelerator.device)
+        model.warmup(verbose=accelerator.is_main_process)
+
+    # Log model info
+    total_params = sum(p.numel() for p in model.parameters())
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    log_message(f"Total params: {total_params:,}", cfg, accelerator)
+    log_message(f"Trainable params: {trainable_params:,}", cfg, accelerator)
+
+    # === Data ===
+    log_message("Creating dataloaders...", cfg, accelerator)
+    dataloaders = create_dataloaders(cfg, tokenizer)
+
+    train_dataloader = dataloaders["train"]
+    val_dataloader = dataloaders.get("validation", None)
+
+    log_message(
+        f"Train dataset size: {len(train_dataloader.dataset)}", cfg, accelerator
+    )
+    log_message(
+        f"Train batches per epoch (before DDP split): {len(train_dataloader)}",
+        cfg,
+        accelerator,
+    )
+
+    if val_dataloader:
+        log_message(
+            f"Validation dataset size: {len(val_dataloader.dataset)}", cfg, accelerator
+        )
+        log_message(f"Validation batches: {len(val_dataloader)}", cfg, accelerator)
+    else:
+        log_message("No validation dataset found", cfg, accelerator)
+
+    # === Optimizer ===
+    log_message("Creating optimizer...", cfg, accelerator)
+    param_groups = group_params(model)
+
+    for group in param_groups:
+        if "lr" not in group:
+            group["lr"] = cfg.training.lr
+        else:
+            group["lr"] = cfg.training.lr * group.get("lr_multiplier", 1.0)
+        if "weight_decay" not in group:
+            group["weight_decay"] = cfg.training.weight_decay
+
+    optimizer = torch.optim.AdamW(
+        param_groups,
+        lr=cfg.training.lr,
+        betas=tuple(cfg.training.betas),
+        eps=cfg.training.eps,
+    )
+
+    # === Scheduler ===
+    steps_per_epoch = math.ceil(len(train_dataloader) / accelerator.num_processes)
+    total_steps = (
+        cfg.training.epochs
+        * steps_per_epoch
+        // cfg.training.gradient_accumulation_steps
+    )
+    scheduler = get_lr_scheduler(optimizer, cfg, total_steps)
+
+    log_message(
+        f"Total steps: {total_steps}, Steps per epoch: {steps_per_epoch}",
+        cfg,
+        accelerator,
+    )
+
+    # === Accelerate Prepare ===
+    log_message(
+        "Preparing model, optimizer, and dataloaders with Accelerate...",
+        cfg,
+        accelerator,
+    )
+
+    if val_dataloader is not None:
+        model, optimizer, train_dataloader, val_dataloader, scheduler = (
+            accelerator.prepare(
+                model, optimizer, train_dataloader, val_dataloader, scheduler
+            )
+        )
+    else:
+        model, optimizer, train_dataloader, scheduler = accelerator.prepare(
+            model, optimizer, train_dataloader, scheduler
+        )
+
+    log_message(
+        f"Train batches per epoch (after DDP split): {len(train_dataloader)}",
+        cfg,
+        accelerator,
+    )
+
+    # === Resume ===
+    global_step = 0
+    start_epoch = 1
+
+    if cfg.training.resume and cfg.training.resume_checkpoint:
+        global_step, start_epoch = load_checkpoint(
+            model,
+            optimizer,
+            scheduler,
+            cfg.training.resume_checkpoint,
+            cfg,
+            accelerator,
+        )
+        start_epoch += 1
+
+    # === Training Loop ===
+    log_message("Starting training...", cfg, accelerator)
+
+    best_val_loss = float("inf")
+
+    try:
+        for epoch in range(start_epoch, cfg.training.epochs + 1):
+            log_message(f"\n{'=' * 60}", cfg, accelerator)
+            log_message(f"EPOCH {epoch}/{cfg.training.epochs}", cfg, accelerator)
+            log_message(f"{'=' * 60}", cfg, accelerator)
+
+            global_step, best_val_loss = train_epoch(
+                model=model,
+                dataloader=train_dataloader,
+                optimizer=optimizer,
+                scheduler=scheduler,
+                cfg=cfg,
+                epoch=epoch,
+                global_step=global_step,
+                accelerator=accelerator,
+                val_dataloader=val_dataloader,
+                best_val_loss=best_val_loss,
+            )
+
+            if cfg.logging.save_every_epoch:
+                save_checkpoint(
+                    model, optimizer, scheduler, global_step, epoch, cfg, accelerator
+                )
+
+    except KeyboardInterrupt:
+        log_message("Training interrupted by user", cfg, accelerator)
+        save_checkpoint(
+            model, optimizer, scheduler, global_step, epoch, cfg, accelerator
+        )
+
+    # === Final Save ===
+    log_message("\nTraining completed!", cfg, accelerator)
+
+    if accelerator.is_main_process:
+        final_model_path = Path(cfg.paths.output_dir) / "model_final.pt"
+        unwrapped_model = accelerator.unwrap_model(model)
+        torch.save(unwrapped_model.state_dict(), final_model_path)
+        log_message(f"Final model: {final_model_path}", cfg, accelerator)
+
+    accelerator.wait_for_everyone()
+    accelerator.end_training()
+    finish_tracking()
+
+
+if __name__ == "__main__":
+    main()

lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/files/config.yaml

@@ -0,0 +1,151 @@
+_wandb:
+    value:
+        cli_version: 0.24.0
+        code_path: code/code_completion_exp/train_hnet/train.py
+        e:
+            dyau2sb185lrum72o3jj1efzj126ey29:
+                args:
+                    - tracking=wandb
+                    - tracking.project=code-completion_lr-sweep
+                    - tracking.run_name=hnet_xl_code_lr_5e-4
+                    - training.lr=5e-4
+                    - paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4
+                    - data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+                codePath: code_completion_exp/train_hnet/train.py
+                codePathLocal: train.py
+                cpu_count: 64
+                cpu_count_logical: 128
+                cudaVersion: "12.2"
+                disk:
+                    /:
+                        total: "265214230528"
+                        used: "170465628160"
+                email: nikita@local.ru
+                executable: /venv/bytellm/bin/python
+                git:
+                    commit: 0a7180b6ab9f63d2794494f09ec4918576d10fa2
+                    remote: https://github.com/naryst/byte-llms-code.git
+                gpu: NVIDIA H100 80GB HBM3
+                gpu_count: 4
+                gpu_nvidia:
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-b60cdcab-2033-2009-41de-be646c953a20
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-9982b420-4520-4238-c378-ec5a46015474
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-e26ebaac-aaa6-3eed-17ab-a3dce303a76f
+                    - architecture: Hopper
+                      cudaCores: 16896
+                      memoryTotal: "85520809984"
+                      name: NVIDIA H100 80GB HBM3
+                      uuid: GPU-9dfc6dba-0be6-4a10-1027-336cc0e65134
+                host: 7504e518d24a
+                memory:
+                    total: "1081679683584"
+                os: Linux-5.4.0-176-generic-x86_64-with-glibc2.35
+                program: /workspace/byte-llms-code/code_completion_exp/train_hnet/train.py
+                python: CPython 3.12.0
+                root: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4
+                startedAt: "2026-04-26T00:19:41.197293Z"
+                writerId: dyau2sb185lrum72o3jj1efzj126ey29
+        m: []
+        python_version: 3.12.0
+        t:
+            "1":
+                - 1
+                - 11
+                - 49
+                - 50
+                - 51
+                - 71
+                - 105
+            "2":
+                - 1
+                - 11
+                - 49
+                - 50
+                - 51
+                - 71
+                - 105
+            "3":
+                - 2
+                - 13
+                - 16
+                - 61
+            "4": 3.12.0
+            "5": 0.24.0
+            "6": 4.57.6
+            "12": 0.24.0
+            "13": linux-x86_64
+data:
+    value:
+        max_context_len: 4096
+        max_target_len: 256
+        max_train_samples: null
+        max_val_samples: 2000
+        num_workers: 0
+        path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+        pin_memory: true
+device:
+    value: cuda
+logging:
+    value:
+        eval_interval: 2000
+        log_interval: 10
+        save_every_epoch: false
+        save_interval: 0
+model:
+    value:
+        checkpoint_path: /workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt
+        config_path: /workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json
+paths:
+    value:
+        output_dir: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4
+seed:
+    value: 42
+tracking:
+    value:
+        backend: wandb
+        base_url: https://wandb.platun0v.ru
+        enabled: true
+        entity: null
+        local_dir: /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4
+        project: code-completion_lr-sweep
+        run_name: hnet_xl_code_lr_5e-4
+training:
+    value:
+        batch_size: 4
+        betas:
+            - 0.9
+            - 0.95
+        decay_ratio: 0.2
+        epochs: 1
+        eps: 1e-08
+        eval_batch_size: 24
+        gradient_accumulation_steps: 4
+        load_balancing_N: 4
+        load_balancing_weight: 0.01
+        lr: 0.0005
+        lr_multiplier:
+            - 2
+            - 1.5
+            - 1
+        lr_scheduler: wsd
+        max_grad_norm: 1
+        min_lr_ratio: 0.1
+        resume: false
+        resume_checkpoint: null
+        use_amp: true
+        warmup_model: true
+        warmup_ratio: 0.1
+        warmup_steps: 100
+        weight_decay: 0.1

lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/files/requirements.txt

@@ -0,0 +1,245 @@
+setuptools==78.1.1
+wheel==0.45.1
+pip==25.2
+webencodings==0.5.1
+triton==3.2.0
+pytz==2025.2
+pydub==0.25.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.6.2
+mpmath==1.3.0
+ipython-genutils==0.2.0
+fastjsonschema==2.21.2
+brotli==1.2.0
+antlr4-python3-runtime==4.9.3
+xxhash==3.6.0
+widgetsnbextension==4.0.14
+websocket-client==1.9.0
+webcolors==24.11.1
+wcwidth==0.2.14
+urllib3==2.5.0
+uri-template==1.3.0
+tzdata==2025.2
+typing_extensions==4.15.0
+types-python-dateutil==2.9.0.20251008
+traitlets==5.14.3
+tqdm==4.67.1
+tornado==6.5.2
+tomlkit==0.13.3
+tinycss2==1.4.0
+tabulate==0.9.0
+sympy==1.13.1
+soupsieve==2.8
+sniffio==1.3.1
+smmap==5.0.2
+six==1.17.0
+shellingham==1.5.4
+Send2Trash==1.8.3
+semantic-version==2.10.0
+safetensors==0.6.2
+rpds-py==0.27.1
+rfc3986-validator==0.1.1
+regex==2025.9.18
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-multipart==0.0.22
+python-json-logger==4.0.0
+python-dotenv==1.2.1
+pyparsing==3.2.5
+PyJWT==2.8.0
+Pygments==2.19.2
+pycparser==2.23
+pyarrow==22.0.0
+psutil==7.1.0
+protobuf==6.33.4
+propcache==0.4.1
+prometheus_client==0.23.1
+portalocker==3.2.0
+platformdirs==4.5.0
+pillow==11.3.0
+pexpect==4.9.0
+pathspec==1.0.4
+parso==0.8.5
+pandocfilters==1.5.1
+packaging==25.0
+orjson==3.11.6
+opt_einsum==3.4.0
+nvidia-nvtx-cu12==12.4.127
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nccl-cu12==2.21.5
+nvidia-curand-cu12==10.3.5.147
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cufft-cu12==11.2.1.3
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cublas-cu12==12.4.5.8
+numpy==2.3.3
+ninja==1.13.0
+networkx==3.5
+nest-asyncio==1.6.0
+narwhals==2.15.0
+mypy_extensions==1.1.0
+multidict==6.7.0
+mistune==3.1.4
+mdurl==0.1.2
+MarkupSafe==3.0.3
+lxml==6.0.2
+librt==0.8.0
+lark==1.3.0
+kiwisolver==1.4.9
+jupyterlab_widgets==3.0.15
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.12.1
+itsdangerous==2.2.0
+idna==3.10
+hf-xet==1.1.10
+h11==0.16.0
+groovy==0.1.2
+fsspec==2025.9.0
+frozenlist==1.8.0
+fqdn==1.5.1
+fonttools==4.60.1
+filelock==3.19.1
+ffmpy==1.0.0
+executing==2.2.1
+einops==0.8.1
+dill==0.4.0
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.17
+dacite==1.9.2
+cycler==0.12.1
+comm==0.2.3
+colorama==0.4.6
+click==8.3.1
+charset-normalizer==3.4.3
+certifi==2025.10.5
+bleach==6.2.0
+babel==2.17.0
+attrs==25.4.0
+async-lru==2.0.5
+asttokens==3.0.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+aiofiles==24.1.0
+yarl==1.22.0
+uvicorn==0.40.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+sentry-sdk==2.50.0
+scipy==1.17.0
+sacrebleu==2.6.0
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+reportlab==4.4.9
+referencing==0.36.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+plotly==6.5.2
+pathlib2==2.3.7.post1
+orderedmultidict==1.0.2
+optree==0.17.0
+omegaconf==2.3.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+mypy==1.19.1
+multiprocess==0.70.16
+matplotlib-inline==0.1.7
+markdown-it-py==4.0.0
+jupyter_core==5.8.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+httpcore==1.0.9
+gitdb==4.0.12
+ftfy==6.3.1
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.2
+anyio==4.11.0
+aiosignal==1.4.0
+starlette==0.50.0
+rich==14.2.0
+pydantic==2.12.5
+pandas==2.3.3
+nvidia-cusolver-cu12==11.6.1.9
+matplotlib==3.10.7
+jupyter_server_terminals==0.5.3
+jupyter_client==8.6.3
+jsonschema-specifications==2025.9.1
+ipython==9.6.0
+hydra-core==1.3.2
+huggingface-hub==0.35.3
+httpx==0.28.1
+GitPython==3.1.46
+furl==2.1.4
+cryptography==46.0.4
+arrow==1.3.0
+argon2-cffi-bindings==25.1.0
+aiohttp==3.13.1
+wandb==0.24.0
+typer==0.21.1
+torch==2.6.0
+tokenizers==0.22.1
+seaborn==0.13.2
+safehttpx==0.1.7
+jsonschema==4.25.1
+joypy==0.2.6
+isoduration==20.11.0
+ipywidgets==8.1.7
+ipykernel==6.30.1
+gradio_client==2.0.3
+fastapi==0.128.0
+Authlib==1.6.6
+argon2-cffi==25.1.0
+transformers==4.57.6
+nbformat==5.10.4
+mlstm_kernels==2.0.2
+jupyter-console==6.6.3
+gradio==6.5.1
+datasets==4.3.0
+clearml==1.16.4
+accelerate==1.10.1
+xlstm==2.0.4
+nbclient==0.10.2
+jupyter-events==0.12.0
+trackio==0.15.0
+nbconvert==7.16.6
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.27.3
+jupyter-lsp==2.3.0
+nbclassic==1.3.3
+jupyterlab==4.4.9
+notebook==7.4.7
+jupyter_contrib_core==0.4.2
+jupyter==1.1.1
+jupyter_nbextensions_configurator==0.6.4
+causal-conv1d==1.5.0.post8
+flash_attn==2.7.4.post1
+mamba-ssm==2.2.4
+hnet==0.0.1
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/files/wandb-metadata.json

@@ -0,0 +1,69 @@
+{
+  "os": "Linux-5.4.0-176-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.12.0",
+  "startedAt": "2026-04-26T00:19:41.197293Z",
+  "args": [
+    "tracking=wandb",
+    "tracking.project=code-completion_lr-sweep",
+    "tracking.run_name=hnet_xl_code_lr_5e-4",
+    "training.lr=5e-4",
+    "paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4",
+    "data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full"
+  ],
+  "program": "/workspace/byte-llms-code/code_completion_exp/train_hnet/train.py",
+  "codePath": "code_completion_exp/train_hnet/train.py",
+  "codePathLocal": "train.py",
+  "git": {
+    "remote": "https://github.com/naryst/byte-llms-code.git",
+    "commit": "0a7180b6ab9f63d2794494f09ec4918576d10fa2"
+  },
+  "email": "nikita@local.ru",
+  "root": "/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4",
+  "host": "7504e518d24a",
+  "executable": "/venv/bytellm/bin/python",
+  "cpu_count": 64,
+  "cpu_count_logical": 128,
+  "gpu": "NVIDIA H100 80GB HBM3",
+  "gpu_count": 4,
+  "disk": {
+    "/": {
+      "total": "265214230528",
+      "used": "170465628160"
+    }
+  },
+  "memory": {
+    "total": "1081679683584"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-b60cdcab-2033-2009-41de-be646c953a20"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-9982b420-4520-4238-c378-ec5a46015474"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-e26ebaac-aaa6-3eed-17ab-a3dce303a76f"
+    },
+    {
+      "name": "NVIDIA H100 80GB HBM3",
+      "memoryTotal": "85520809984",
+      "cudaCores": 16896,
+      "architecture": "Hopper",
+      "uuid": "GPU-9dfc6dba-0be6-4a10-1027-336cc0e65134"
+    }
+  ],
+  "cudaVersion": "12.2",
+  "writerId": "dyau2sb185lrum72o3jj1efzj126ey29"
+}

lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/step_time":0.7018253087997437,"best/val_perplexity":1.3968818580448776,"_runtime":7135,"train/soft_boundary_ratio_stage0":0.3369161636344325,"epoch/hard_boundary_ratio_stage0":0.35071097942532414,"_timestamp":1.7771699152059996e+09,"train/loss_avg":0.35818642927241806,"epoch/chunk_len_stage1":2.6923843128064373,"train/lb_loss":1.0790050029754639,"best/val_loss":0.34870333082619165,"train/loss":0.24225624278187752,"epoch/lm_loss":0.3488054759618164,"val/lm_loss":0.33815049202669234,"epoch/soft_boundary_ratio_stage0":0.3369219771080528,"epoch/loss":0.3581828390180251,"train/hard_boundary_ratio_stage0":0.3507020525501328,"val/time":4.524768352508545,"train/lm_loss":0.2673550844192505,"best/step":8000,"train/chunk_len_stage1":2.692232057831329,"val/loss":0.34870333082619165,"epoch/soft_boundary_ratio_stage1":0.36771885260966747,"train/hard_boundary_ratio_stage1":0.3750683158667268,"epoch/lb_loss":1.0680555066867534,"epoch/chunk_len_stage0":2.873913584180546,"train/lr":5e-05,"train/epoch":1,"val/perplexity":1.3968818580448776,"val/lb_loss":1.0552839296204704,"_wandb":{"runtime":7135},"_step":9880,"train/chunk_len_stage0":2.8739849724727216,"epoch/time":7083.666541099548,"train/soft_boundary_ratio_stage1":0.3677366381526144,"epoch/hard_boundary_ratio_stage1":0.3750465785576219}

lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2026-04-26T00:19:41.280606372Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpj_w4xo0a/port-257611.txt","pid":257611,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-26T00:19:41.281310681Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":257611}
+{"time":"2026-04-26T00:19:41.281295344Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-257611-257664-4135181244/socket","Net":"unix"}}
+{"time":"2026-04-26T00:19:41.468587343Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-26T00:19:41.489622086Z","level":"INFO","msg":"handleInformInit: received","streamId":"i6lt8av0","id":"1(@)"}
+{"time":"2026-04-26T00:19:41.862752556Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"i6lt8av0","id":"1(@)"}
+{"time":"2026-04-26T02:18:39.038091992Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"i6lt8av0","id":"1(@)"}
+{"time":"2026-04-26T02:18:39.041417993Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"i6lt8av0","id":"1(@)"}
+{"time":"2026-04-26T02:18:39.05126405Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-26T02:18:39.051282767Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-26T02:18:39.051287843Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-26T02:18:39.051291659Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-26T02:18:39.051337316Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-26T02:18:39.051357455Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-26T02:18:39.051349453Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-257611-257664-4135181244/socket","Net":"unix"}}
+{"time":"2026-04-26T02:18:39.051367174Z","level":"INFO","msg":"server is closed"}

lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/logs/debug-internal.log

@@ -0,0 +1,13 @@
+{"time":"2026-04-26T00:19:41.489729754Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-04-26T00:19:41.862602753Z","level":"INFO","msg":"stream: created new stream","id":"i6lt8av0"}
+{"time":"2026-04-26T00:19:41.862649982Z","level":"INFO","msg":"handler: started","stream_id":"i6lt8av0"}
+{"time":"2026-04-26T00:19:41.862747075Z","level":"INFO","msg":"stream: started","id":"i6lt8av0"}
+{"time":"2026-04-26T00:19:41.862757167Z","level":"INFO","msg":"writer: started","stream_id":"i6lt8av0"}
+{"time":"2026-04-26T00:19:41.862759475Z","level":"INFO","msg":"sender: started","stream_id":"i6lt8av0"}
+{"time":"2026-04-26T00:19:41.985346468Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-04-26T02:18:38.932345179Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-26T02:18:39.035750391Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-04-26T02:18:39.038119733Z","level":"INFO","msg":"stream: closing","id":"i6lt8av0"}
+{"time":"2026-04-26T02:18:39.038132189Z","level":"INFO","msg":"handler: closed","stream_id":"i6lt8av0"}
+{"time":"2026-04-26T02:18:39.038193257Z","level":"INFO","msg":"sender: closed","stream_id":"i6lt8av0"}
+{"time":"2026-04-26T02:18:39.038198898Z","level":"INFO","msg":"stream: closed","id":"i6lt8av0"}

lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/logs/debug.log

@@ -0,0 +1,24 @@
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_setup.py:_flush():81] Configure stats pid to 257611
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_init.py:setup_run_log_directory():717] Logging user logs to /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/logs/debug.log
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to /workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/logs/debug-internal.log
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_init.py:init():844] calling init triggers
+2026-04-26 00:19:41,198 INFO    MainThread:257611 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'config_path': '/workspace/byte-llms-code/hnet_project/configs/hnet_2stage_XL_code.json', 'checkpoint_path': '/workspace/byte-llms-code/hnet_project/checkpoints/hnet_2stage_XL_code.pt'}, 'training': {'epochs': 1, 'batch_size': 4, 'eval_batch_size': 24, 'gradient_accumulation_steps': 4, 'lr': 0.0005, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'lr_multiplier': [2.0, 1.5, 1.0], 'load_balancing_weight': 0.01, 'load_balancing_N': 4.0, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None, 'warmup_model': True}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 0, 'pin_memory': True, 'max_train_samples': None, 'max_val_samples': 2000}, 'logging': {'log_interval': 10, 'save_interval': 0, 'eval_interval': 2000, 'save_every_epoch': False}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'code-completion_lr-sweep', 'run_name': 'hnet_xl_code_lr_5e-4', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': '/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4'}, 'paths': {'output_dir': '/workspace/byte-llms-code/outputs/lr_sweep/hnet_xl_code_lr_5e-4'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/code_completion_exp/train_hnet/train.py'}}
+2026-04-26 00:19:41,199 INFO    MainThread:257611 [wandb_init.py:init():892] starting backend
+2026-04-26 00:19:41,468 INFO    MainThread:257611 [wandb_init.py:init():895] sending inform_init request
+2026-04-26 00:19:41,488 INFO    MainThread:257611 [wandb_init.py:init():903] backend started and connected
+2026-04-26 00:19:41,491 INFO    MainThread:257611 [wandb_init.py:init():973] updated telemetry
+2026-04-26 00:19:41,508 INFO    MainThread:257611 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-26 00:19:41,984 INFO    MainThread:257611 [wandb_init.py:init():1044] starting run threads in backend
+2026-04-26 00:19:42,142 INFO    MainThread:257611 [wandb_run.py:_console_start():2529] atexit reg
+2026-04-26 00:19:42,142 INFO    MainThread:257611 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-04-26 00:19:42,142 INFO    MainThread:257611 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-04-26 00:19:42,142 INFO    MainThread:257611 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-04-26 00:19:42,145 INFO    MainThread:257611 [wandb_init.py:init():1084] run started, returning control to user process
+2026-04-26 02:18:37,753 INFO    MainThread:257611 [wandb_run.py:_finish():2295] finishing run nikita/code-completion_lr-sweep/i6lt8av0
+2026-04-26 02:18:37,754 INFO    MainThread:257611 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-04-26 02:18:37,754 INFO    MainThread:257611 [wandb_run.py:_restore():2476] restore
+2026-04-26 02:18:37,754 INFO    MainThread:257611 [wandb_run.py:_restore():2482] restore done
+2026-04-26 02:18:39,037 INFO    MainThread:257611 [wandb_run.py:_footer_sync_info():3870] logging synced files

lr_sweep/hnet_xl_code_lr_5e-4/wandb/run-20260426_001941-i6lt8av0/run-i6lt8av0.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4904ef01e885208188a41b5d7f41a4a1759d924be32288783eb848b49f89430b
+size 3065241

lr_sweep/hnet_xl_code_lr_5e-5/model_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:310965a6147f6c3900fdda4b4054876d929624a57099569b4c715c336af405ec
+size 3315165139

lr_sweep/pythia_1b_lr_1e-4/.hydra/config.yaml

@@ -0,0 +1,49 @@
+model:
+  name: EleutherAI/pythia-1b
+  checkpoint_path: null
+  from_scratch: false
+training:
+  epochs: 1
+  batch_size: 4
+  eval_batch_size: 12
+  gradient_accumulation_steps: 4
+  lr: 0.0001
+  weight_decay: 0.1
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-08
+  lr_scheduler: wsd
+  warmup_ratio: 0.1
+  decay_ratio: 0.2
+  warmup_steps: 100
+  min_lr_ratio: 0.1
+  max_grad_norm: 1.0
+  use_amp: true
+  resume: false
+  resume_checkpoint: null
+data:
+  path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 4
+  pin_memory: true
+  max_train_samples: null
+  max_val_samples: 2000
+logging:
+  log_interval: 10
+  save_interval: 0
+  eval_interval: 2000
+  save_every_epoch: false
+tracking:
+  enabled: true
+  backend: wandb
+  project: code-completion_lr-sweep
+  run_name: pythia_1b_lr_1e-4
+  entity: null
+  base_url: https://wandb.platun0v.ru
+  local_dir: ${paths.output_dir}
+paths:
+  output_dir: /workspace/byte-llms-code/outputs/lr_sweep/pythia_1b_lr_1e-4
+seed: 42
+device: cuda

lr_sweep/pythia_1b_lr_1e-4/.hydra/hydra.yaml

@@ -0,0 +1,167 @@
+hydra:
+  run:
+    dir: ${paths.output_dir}
+  sweep:
+    dir: outputs/multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - tracking=wandb
+    - tracking.project=code-completion_lr-sweep
+    - tracking.run_name=pythia_1b_lr_1e-4
+    - training.lr=1e-4
+    - paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/pythia_1b_lr_1e-4
+    - model=pythia_1b
+    - data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+  job:
+    name: train
+    chdir: false
+    override_dirname: data.path=/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full,model=pythia_1b,paths.output_dir=/workspace/byte-llms-code/outputs/lr_sweep/pythia_1b_lr_1e-4,tracking.project=code-completion_lr-sweep,tracking.run_name=pythia_1b_lr_1e-4,tracking=wandb,training.lr=1e-4
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace/byte-llms-code/code_completion_exp/train_pythia
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/byte-llms-code/code_completion_exp/train_pythia/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/byte-llms-code/outputs/lr_sweep/pythia_1b_lr_1e-4
+    choices:
+      paths: default
+      tracking: wandb
+      logging: default
+      data: default
+      training: default
+      model: pythia_1b
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false