Add files using upload-large-folder tool

.gitattributes

@@ -46,3 +46,4 @@ lr_sweep/hnet_xl_code_lr_1e-4/wandb/run-20260425_200722-d5usyud5/run-d5usyud5.wa
 lr_sweep/pythia_1b_lr_1e-4/wandb/run-20260425_201333-p8ozhgpm/run-p8ozhgpm.wandb filter=lfs diff=lfs merge=lfs -text
 lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_220653-ln6tfunh/run-ln6tfunh.wandb filter=lfs diff=lfs merge=lfs -text
 lr_sweep/hnet_xl_code_lr_2e-4/wandb/run-20260425_222011-khn25dwv/run-khn25dwv.wandb filter=lfs diff=lfs merge=lfs -text
+pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/run-jhqe4qjw.wandb filter=lfs diff=lfs merge=lfs -text

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/.hydra/config.yaml

@@ -0,0 +1,49 @@
+model:
+  name: EleutherAI/pythia-1.4b
+  checkpoint_path: null
+  from_scratch: false
+training:
+  epochs: 3
+  batch_size: 4
+  eval_batch_size: 12
+  gradient_accumulation_steps: 4
+  lr: 2.0e-05
+  weight_decay: 0.1
+  betas:
+  - 0.9
+  - 0.95
+  eps: 1.0e-08
+  lr_scheduler: wsd
+  warmup_ratio: 0.1
+  decay_ratio: 0.2
+  warmup_steps: 100
+  min_lr_ratio: 0.1
+  max_grad_norm: 1.0
+  use_amp: true
+  resume: false
+  resume_checkpoint: null
+data:
+  path: ${oc.env:PROJECT_ROOT}/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 4
+  pin_memory: true
+  max_train_samples: null
+  max_val_samples: null
+logging:
+  log_interval: 10
+  save_interval: 6000
+  eval_interval: 1000
+  save_every_epoch: true
+tracking:
+  enabled: true
+  backend: wandb
+  project: code-completion_pythia-1.4b-rerun
+  run_name: pythia_1_4b_lr_2e-5
+  entity: null
+  base_url: https://wandb.platun0v.ru
+  local_dir: ${paths.output_dir}
+paths:
+  output_dir: outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5
+seed: 42
+device: cuda

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/.hydra/hydra.yaml

@@ -0,0 +1,160 @@
+hydra:
+  run:
+    dir: ${paths.output_dir}
+  sweep:
+    dir: outputs/multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task: []
+  job:
+    name: train
+    chdir: false
+    override_dirname: ''
+    id: ???
+    num: ???
+    config_name: config
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.3'
+    cwd: /workspace/byte-llms-code/code_completion_exp/train_pythia
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /workspace/byte-llms-code/code_completion_exp/train_pythia/configs
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /workspace/byte-llms-code/code_completion_exp/train_pythia/outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5
+    choices:
+      paths: default
+      tracking: wandb
+      logging: default
+      data: default
+      training: default
+      model: pythia_1_4b
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/.hydra/overrides.yaml

@@ -0,0 +1 @@
+[]

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/checkpoints/checkpoint_latest.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:619099feb68a23252ee1d9d123dc5e80777d59a517d2a1277e16b11080ff840d
+size 8488268654

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/checkpoints/checkpoint_step_18000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94e3195e5d668f03e57524372992bcc7ad140118ed7201e0616464b3ee3a851e
+size 8488273342

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/checkpoints/checkpoint_step_19774.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5cda16b18df8729407d83bd272355d7d621d43adccbbf1b5c643452c224677f
+size 8488273342

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/checkpoints/checkpoint_step_24000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:832b215b90a6d53e33a0548db8f8eabc715240f2f1f78fdf7e5885e8b469462a
+size 8488273342

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/checkpoints/checkpoint_step_29661.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2619e2e59f770c5a99cfb4b12e9b25be6042452912ebc49a6f8e366641bac9bc
+size 8488273342

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/checkpoints/checkpoint_step_6000.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49c762d0ff93a76e555f3cb446b85b195af76e053559fbe8fc21b7e2e5b7d297
+size 8488272170

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/.ipynb_checkpoints/summary-checkpoint.txt

@@ -0,0 +1,16 @@
+EVALUATION SUMMARY
+==================================================================================================
+
+Checkpoint                      Exact Match    Token Acc     BLEU   PERPLEXITY  ms/sample   samp/s
+--------------------------------------------------------------------------------------------------
+initial_checkpoint                    0.00%       28.41%     1.10       537.69       33.0    30.34
+checkpoint_latest                    34.91%       36.09%    19.27       318.72       13.5    74.14
+checkpoint_step_12000                33.88%       35.22%    18.91       291.07       13.3    75.07
+checkpoint_step_18000                34.98%       36.06%    19.55       297.06       13.5    74.32
+checkpoint_step_19774                35.04%       36.08%    19.47       297.00       13.3    74.97
+checkpoint_step_24000                34.82%       36.00%    19.17       313.80       13.6    73.32
+checkpoint_step_29661                34.81%       36.03%    19.32       318.72       13.5    74.20
+checkpoint_step_6000                 32.96%       34.80%    18.37       289.31       13.5    74.03
+checkpoint_step_9887                 33.98%       35.17%    18.42       268.97       13.6    73.46
+model_best                           35.00%       36.02%    19.48       294.19       13.3    74.91
+model_final                          34.89%       36.07%    19.29       318.72       13.5    74.09

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/eval_config.yaml

@@ -0,0 +1,31 @@
+data:
+  path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+  max_context_len: 4096
+  max_target_len: 256
+  num_workers: 4
+  pin_memory: true
+  max_train_samples: null
+  max_val_samples: null
+model:
+  name: EleutherAI/pythia-1.4b
+  checkpoint_path: null
+  from_scratch: false
+paths:
+  checkpoints_dir: outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5
+  initial_checkpoint: auto
+  output_dir: outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results
+evaluation:
+  batch_size: 16
+  max_samples: null
+  compute_perplexity: true
+  bleu_tokenize: none
+  save_predictions: true
+  use_amp: true
+generation:
+  max_new_tokens: 64
+  temperature: 0.1
+  top_k: 0
+  top_p: 1.0
+  do_sample: true
+seed: 42
+device: cuda

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_checkpoint_latest.txt

@@ -0,0 +1,17 @@
+Checkpoint: checkpoint_latest.pt
+================================================================================
+
+exact_match: 0.3491082112940908
+token_accuracy: 0.3609102339784317
+bleu: 19.269757348478024
+perplexity: 318.723480592771
+num_samples: 35098
+gen_wall_time_s: 473.41876831650734
+gen_samples_per_s: 74.13732270228668
+gen_time_per_sample_ms: 13.48848277156839
+gen_chars_per_s: 1919.0747405954103
+gen_batch_mean_ms: 417.0968242689166
+gen_batch_p50_ms: 357.9665273427963
+gen_batch_p95_ms: 846.3990613818174
+gen_batch_max_ms: 3587.848901748657
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_checkpoint_step_12000.txt

@@ -0,0 +1,17 @@
+Checkpoint: checkpoint_step_12000.pt
+================================================================================
+
+exact_match: 0.33879423328964614
+token_accuracy: 0.3521884334688284
+bleu: 18.905961886825054
+perplexity: 291.0708305846913
+num_samples: 35098
+gen_wall_time_s: 467.52853602170944
+gen_samples_per_s: 75.07135350208921
+gen_time_per_sample_ms: 13.320660323144038
+gen_chars_per_s: 1932.4296388147054
+gen_batch_mean_ms: 416.61676505721476
+gen_batch_p50_ms: 356.2297746539116
+gen_batch_p95_ms: 898.9157944917679
+gen_batch_max_ms: 3586.8209674954414
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_checkpoint_step_18000.txt

@@ -0,0 +1,17 @@
+Checkpoint: checkpoint_step_18000.pt
+================================================================================
+
+exact_match: 0.34982050259274033
+token_accuracy: 0.3606308170284673
+bleu: 19.550134976587582
+perplexity: 297.0567970765463
+num_samples: 35098
+gen_wall_time_s: 472.2754218801856
+gen_samples_per_s: 74.31680408070065
+gen_time_per_sample_ms: 13.455906942851035
+gen_chars_per_s: 1911.1178735635738
+gen_batch_mean_ms: 411.898644021292
+gen_batch_p50_ms: 349.4953215122223
+gen_batch_p95_ms: 808.1516057252898
+gen_batch_max_ms: 3586.3386392593384
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_checkpoint_step_19774.txt

@@ -0,0 +1,17 @@
+Checkpoint: checkpoint_step_19774.pt
+================================================================================
+
+exact_match: 0.3504188272836059
+token_accuracy: 0.3608470325254635
+bleu: 19.467009802659206
+perplexity: 296.99746886537366
+num_samples: 35098
+gen_wall_time_s: 468.18471866846085
+gen_samples_per_s: 74.96613751046883
+gen_time_per_sample_ms: 13.33935605072827
+gen_chars_per_s: 1926.4063179274315
+gen_batch_mean_ms: 415.29658690108533
+gen_batch_p50_ms: 353.5846024751663
+gen_batch_p95_ms: 864.5775735378279
+gen_batch_max_ms: 3583.6338102817535
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_checkpoint_step_24000.txt

@@ -0,0 +1,17 @@
+Checkpoint: checkpoint_step_24000.pt
+================================================================================
+
+exact_match: 0.34822497008376546
+token_accuracy: 0.35996553857617103
+bleu: 19.169750619802162
+perplexity: 313.79867716386906
+num_samples: 35098
+gen_wall_time_s: 478.67056403309107
+gen_samples_per_s: 73.32391552193636
+gen_time_per_sample_ms: 13.63811510721668
+gen_chars_per_s: 1899.4065403552715
+gen_batch_mean_ms: 415.3155261547107
+gen_batch_p50_ms: 358.4626540541649
+gen_batch_p95_ms: 833.3663418889046
+gen_batch_max_ms: 3585.671544075012
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_checkpoint_step_29661.txt

@@ -0,0 +1,17 @@
+Checkpoint: checkpoint_step_29661.pt
+================================================================================
+
+exact_match: 0.3480540201720896
+token_accuracy: 0.3602582610951814
+bleu: 19.317169450326166
+perplexity: 318.723480592771
+num_samples: 35098
+gen_wall_time_s: 472.98777021467686
+gen_samples_per_s: 74.20487845609608
+gen_time_per_sample_ms: 13.476202923661658
+gen_chars_per_s: 1923.102577445405
+gen_batch_mean_ms: 412.44772313639805
+gen_batch_p50_ms: 353.74465584754944
+gen_batch_p95_ms: 786.1013039946557
+gen_batch_max_ms: 3586.092635989189
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_checkpoint_step_6000.txt

@@ -0,0 +1,17 @@
+Checkpoint: checkpoint_step_6000.pt
+================================================================================
+
+exact_match: 0.3296484130149866
+token_accuracy: 0.34800383200388524
+bleu: 18.371122658286087
+perplexity: 289.313993391532
+num_samples: 35098
+gen_wall_time_s: 474.1001447662711
+gen_samples_per_s: 74.03077258561719
+gen_time_per_sample_ms: 13.507896312219247
+gen_chars_per_s: 1912.9207405053733
+gen_batch_mean_ms: 418.14548702895263
+gen_batch_p50_ms: 347.5266695022583
+gen_batch_p95_ms: 911.2774878740311
+gen_batch_max_ms: 2318.4452056884766
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_checkpoint_step_9887.txt

@@ -0,0 +1,17 @@
+Checkpoint: checkpoint_step_9887.pt
+================================================================================
+
+exact_match: 0.3398199327597014
+token_accuracy: 0.3516528843147299
+bleu: 18.415097468391597
+perplexity: 268.97285058053575
+num_samples: 35098
+gen_wall_time_s: 477.7747625410557
+gen_samples_per_s: 73.46139384451892
+gen_time_per_sample_ms: 13.61259224289292
+gen_chars_per_s: 1895.455915635939
+gen_batch_mean_ms: 433.3023355837399
+gen_batch_p50_ms: 356.7342534661293
+gen_batch_p95_ms: 910.7078820466995
+gen_batch_max_ms: 3595.522381365299
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_initial_checkpoint.txt

@@ -0,0 +1,17 @@
+Checkpoint: initial_checkpoint
+================================================================================
+
+exact_match: 0.0
+token_accuracy: 0.2841204686221418
+bleu: 1.1002232570475041
+perplexity: 537.6947490846742
+num_samples: 35098
+gen_wall_time_s: 1157.0034177526832
+gen_samples_per_s: 30.33526043351967
+gen_time_per_sample_ms: 32.96493867891854
+gen_chars_per_s: 6686.615511496883
+gen_batch_mean_ms: 1047.8846134453224
+gen_batch_p50_ms: 919.267863035202
+gen_batch_p95_ms: 1635.0471004843712
+gen_batch_max_ms: 3586.4143297076225
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_model_best.txt

@@ -0,0 +1,17 @@
+Checkpoint: model_best.pt
+================================================================================
+
+exact_match: 0.34996296085247025
+token_accuracy: 0.36019173324995174
+bleu: 19.47680016321815
+perplexity: 294.1937527039324
+num_samples: 35098
+gen_wall_time_s: 468.51309882104397
+gen_samples_per_s: 74.91359385323449
+gen_time_per_sample_ms: 13.348712143741636
+gen_chars_per_s: 1920.1085354149618
+gen_batch_mean_ms: 414.4478002894218
+gen_batch_p50_ms: 353.60363125801086
+gen_batch_p95_ms: 818.3927312493347
+gen_batch_max_ms: 3586.1896350979805
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/metrics_model_final.txt

@@ -0,0 +1,17 @@
+Checkpoint: model_final.pt
+================================================================================
+
+exact_match: 0.348880278078523
+token_accuracy: 0.36068736569691245
+bleu: 19.290218865092186
+perplexity: 318.723480592771
+num_samples: 35098
+gen_wall_time_s: 473.6957869678736
+gen_samples_per_s: 74.09396698387857
+gen_time_per_sample_ms: 13.496375490565661
+gen_chars_per_s: 1917.0193718898054
+gen_batch_mean_ms: 417.4575515994726
+gen_batch_p50_ms: 359.1250032186508
+gen_batch_p95_ms: 807.6998949050917
+gen_batch_max_ms: 3586.7618024349213
+gen_num_batches: 1097

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/eval_results/summary.txt

@@ -0,0 +1,16 @@
+EVALUATION SUMMARY
+==================================================================================================
+
+Checkpoint                      Exact Match    Token Acc     BLEU   PERPLEXITY  ms/sample   samp/s
+--------------------------------------------------------------------------------------------------
+initial_checkpoint                    0.00%       28.41%     1.10       537.69       33.0    30.34
+checkpoint_latest                    34.91%       36.09%    19.27       318.72       13.5    74.14
+checkpoint_step_12000                33.88%       35.22%    18.91       291.07       13.3    75.07
+checkpoint_step_18000                34.98%       36.06%    19.55       297.06       13.5    74.32
+checkpoint_step_19774                35.04%       36.08%    19.47       297.00       13.3    74.97
+checkpoint_step_24000                34.82%       36.00%    19.17       313.80       13.6    73.32
+checkpoint_step_29661                34.81%       36.03%    19.32       318.72       13.5    74.20
+checkpoint_step_6000                 32.96%       34.80%    18.37       289.31       13.5    74.03
+checkpoint_step_9887                 33.98%       35.17%    18.42       268.97       13.6    73.46
+model_best                           35.00%       36.02%    19.48       294.19       13.3    74.91
+model_final                          34.89%       36.07%    19.29       318.72       13.5    74.09

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/model_best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30322219874e8a6f3736e61b133825d39f1e585a64560fe6e27084f2ff995e06
+size 2829410658

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/model_final.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a446b0ff6ad1f48b5c1989291f67e33fdb25f00ed0575315930f468fc6f4f881
+size 2829410954

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2026-04-25T21:56:04.515355828Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-04-25T21:56:04.914734923Z","level":"INFO","msg":"stream: created new stream","id":"jhqe4qjw"}
+{"time":"2026-04-25T21:56:04.91479529Z","level":"INFO","msg":"handler: started","stream_id":"jhqe4qjw"}
+{"time":"2026-04-25T21:56:04.914888584Z","level":"INFO","msg":"stream: started","id":"jhqe4qjw"}
+{"time":"2026-04-25T21:56:04.914893063Z","level":"INFO","msg":"writer: started","stream_id":"jhqe4qjw"}
+{"time":"2026-04-25T21:56:04.914903421Z","level":"INFO","msg":"sender: started","stream_id":"jhqe4qjw"}
+{"time":"2026-04-25T21:56:05.060789967Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-04-25T23:20:15.280635585Z","level":"ERROR","msg":"api: HTTP error","status":403,"method":"POST","url":"https://wandb.platun0v.ru/files/nikita/code-completion_pythia-1.4b-rerun/jhqe4qjw/file_stream"}
+{"time":"2026-04-25T23:20:15.280696408Z","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 403 Forbidden url=https://wandb.platun0v.ru/files/nikita/code-completion_pythia-1.4b-rerun/jhqe4qjw/file_stream: "}
+{"time":"2026-04-26T01:44:27.121098166Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-26T01:44:27.121923843Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-04-26T01:44:27.124737334Z","level":"INFO","msg":"stream: closing","id":"jhqe4qjw"}
+{"time":"2026-04-26T01:44:27.124749132Z","level":"INFO","msg":"handler: closed","stream_id":"jhqe4qjw"}
+{"time":"2026-04-26T01:44:27.124842763Z","level":"INFO","msg":"sender: closed","stream_id":"jhqe4qjw"}
+{"time":"2026-04-26T01:44:27.124848143Z","level":"INFO","msg":"stream: closed","id":"jhqe4qjw"}

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/debug.log

@@ -0,0 +1,24 @@
+2026-04-25 21:56:04,224 INFO    MainThread:185429 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-04-25 21:56:04,224 INFO    MainThread:185429 [wandb_setup.py:_flush():81] Configure stats pid to 185429
+2026-04-25 21:56:04,224 INFO    MainThread:185429 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-25 21:56:04,225 INFO    MainThread:185429 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/logs/debug.log
+2026-04-25 21:56:04,225 INFO    MainThread:185429 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/logs/debug-internal.log
+2026-04-25 21:56:04,225 INFO    MainThread:185429 [wandb_init.py:init():844] calling init triggers
+2026-04-25 21:56:04,225 INFO    MainThread:185429 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'name': 'EleutherAI/pythia-1.4b', 'checkpoint_path': None, 'from_scratch': False}, 'training': {'epochs': 3, 'batch_size': 4, 'eval_batch_size': 12, 'gradient_accumulation_steps': 4, 'lr': 2e-05, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 4, 'pin_memory': True, 'max_train_samples': None, 'max_val_samples': None}, 'logging': {'log_interval': 10, 'save_interval': 6000, 'eval_interval': 1000, 'save_every_epoch': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'code-completion_pythia-1.4b-rerun', 'run_name': 'pythia_1_4b_lr_2e-5', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5'}, 'paths': {'output_dir': 'outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/code_completion_exp/train_pythia/train.py'}}
+2026-04-25 21:56:04,225 INFO    MainThread:185429 [wandb_init.py:init():892] starting backend
+2026-04-25 21:56:04,496 INFO    MainThread:185429 [wandb_init.py:init():895] sending inform_init request
+2026-04-25 21:56:04,514 INFO    MainThread:185429 [wandb_init.py:init():903] backend started and connected
+2026-04-25 21:56:04,517 INFO    MainThread:185429 [wandb_init.py:init():973] updated telemetry
+2026-04-25 21:56:04,531 INFO    MainThread:185429 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-25 21:56:05,060 INFO    MainThread:185429 [wandb_init.py:init():1044] starting run threads in backend
+2026-04-25 21:56:05,217 INFO    MainThread:185429 [wandb_run.py:_console_start():2529] atexit reg
+2026-04-25 21:56:05,217 INFO    MainThread:185429 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-04-25 21:56:05,217 INFO    MainThread:185429 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-04-25 21:56:05,217 INFO    MainThread:185429 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-04-25 21:56:05,220 INFO    MainThread:185429 [wandb_init.py:init():1084] run started, returning control to user process
+2026-04-26 01:44:26,160 INFO    MainThread:185429 [wandb_run.py:_finish():2295] finishing run nikita/code-completion_pythia-1.4b-rerun/jhqe4qjw
+2026-04-26 01:44:26,160 INFO    MainThread:185429 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-04-26 01:44:26,161 INFO    MainThread:185429 [wandb_run.py:_restore():2476] restore
+2026-04-26 01:44:26,161 INFO    MainThread:185429 [wandb_run.py:_restore():2482] restore done
+2026-04-26 01:44:27,124 INFO    MainThread:185429 [wandb_run.py:_footer_sync_info():3870] logging synced files

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/files/code/code_completion_exp/train_pythia/train.py

@@ -0,0 +1,606 @@
+"""
+Training Pipeline для Pythia (decoder-only transformer) на задаче Code Completion.
+
+Конфигурация через Hydra + OmegaConf, логирование в Trackio.
+Поддержка DDP через Accelerate для multi-GPU тренировки.
+
+Использование:
+    # Базовый запуск (single GPU)
+    python train.py
+
+    # Multi-GPU с Accelerate
+    accelerate launch train.py
+
+    # Multi-GPU с указанием количества GPU
+    accelerate launch --num_processes=4 train.py
+
+    # Переопределение параметров через CLI
+    python train.py training.lr=1e-4 training.epochs=5
+
+    # Выбор другого конфига модели
+    python train.py model=pythia_160m
+
+    # Multirun (sweep)
+    python train.py --multirun training.lr=1e-4,3e-4,1e-3
+
+    # Без логирования
+    python train.py tracking.enabled=false
+"""
+
+import os
+import math
+import time
+from pathlib import Path
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import DataLoader
+from datasets import load_from_disk
+
+import hydra
+from hydra.core.hydra_config import HydraConfig
+from omegaconf import DictConfig, OmegaConf
+from transformers import (
+    AutoTokenizer,
+    AutoModelForCausalLM,
+    AutoConfig,
+    PreTrainedTokenizerBase,
+)
+from accelerate import Accelerator
+from accelerate.utils import set_seed as accelerate_set_seed
+
+# Ensure repo root is on sys.path (needed when running from subdirectory)
+import sys
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+# Shared training library
+from training_lib.utils import AverageMeter, log_message
+from training_lib.checkpointing import save_checkpoint, load_checkpoint
+from training_lib.schedulers import get_lr_scheduler
+from training_lib.tracking import init_tracking, log_metrics, finish_tracking
+from training_lib.validation import run_validation
+
+
+# ============================================================================
+# ДАННЫЕ
+# ============================================================================
+
+
+class CodeCompletionCollator:
+    """Collate function для батчирования примеров code completion."""
+
+    def __init__(
+        self,
+        tokenizer: PreTrainedTokenizerBase,
+        max_context_len: int = 1024,
+        max_target_len: int = 256,
+    ):
+        self.tokenizer = tokenizer
+        self.max_context_len = max_context_len
+        self.max_target_len = max_target_len
+        self.pad_token_id = tokenizer.pad_token_id
+
+    def __call__(self, batch: list[dict]) -> dict:
+        contexts = [item["context"] for item in batch]
+        targets = [item["target"] for item in batch]
+
+        encoded_contexts = self.tokenizer(
+            contexts,
+            add_special_tokens=True,
+            truncation=True,
+            max_length=self.max_context_len,
+            return_tensors=None,
+        )
+        encoded_targets = self.tokenizer(
+            targets,
+            add_special_tokens=False,
+            truncation=True,
+            max_length=self.max_target_len,
+            return_tensors=None,
+        )
+
+        input_ids_list = []
+        context_lengths = []
+
+        for ctx_ids, tgt_ids in zip(
+            encoded_contexts["input_ids"], encoded_targets["input_ids"]
+        ):
+            tgt_ids = tgt_ids + [self.tokenizer.eos_token_id]
+            context_lengths.append(len(ctx_ids))
+            input_ids_list.append(ctx_ids + tgt_ids)
+
+        max_len = max(len(ids) for ids in input_ids_list)
+
+        padded_input_ids = []
+        attention_mask = []
+
+        for ids in input_ids_list:
+            padding_len = max_len - len(ids)
+            padded_input_ids.append(ids + [self.pad_token_id] * padding_len)
+            attention_mask.append([1] * len(ids) + [0] * padding_len)
+
+        return {
+            "input_ids": torch.tensor(padded_input_ids, dtype=torch.long),
+            "attention_mask": torch.tensor(attention_mask, dtype=torch.long),
+            "context_lengths": torch.tensor(context_lengths, dtype=torch.long),
+        }
+
+
+def create_dataloaders(
+    cfg: DictConfig, tokenizer: PreTrainedTokenizerBase
+) -> dict[str, DataLoader]:
+    """Создание DataLoader'ов для train и validation."""
+    dataset_dict = load_from_disk(cfg.data.path)
+
+    collator = CodeCompletionCollator(
+        tokenizer=tokenizer,
+        max_context_len=cfg.data.max_context_len,
+        max_target_len=cfg.data.max_target_len,
+    )
+
+    dataloaders = {}
+
+    if "train" in dataset_dict:
+        train_dataset = dataset_dict["train"]
+        max_train = cfg.data.get("max_train_samples", None)
+        if max_train is not None:
+            train_dataset = train_dataset.select(range(min(max_train, len(train_dataset))))
+        dataloaders["train"] = DataLoader(
+            train_dataset,
+            batch_size=cfg.training.batch_size,
+            shuffle=True,
+            collate_fn=collator,
+            num_workers=cfg.data.num_workers,
+            pin_memory=cfg.data.pin_memory,
+        )
+
+    if "validation" in dataset_dict:
+        val_dataset = dataset_dict["validation"]
+        max_val = cfg.data.get("max_val_samples", None)
+        if max_val is not None:
+            val_dataset = val_dataset.select(range(min(max_val, len(val_dataset))))
+        eval_batch_size = cfg.training.get("eval_batch_size", cfg.training.batch_size)
+        dataloaders["validation"] = DataLoader(
+            val_dataset,
+            batch_size=eval_batch_size,
+            shuffle=False,
+            collate_fn=collator,
+            num_workers=cfg.data.num_workers,
+            pin_memory=cfg.data.pin_memory,
+        )
+
+    return dataloaders
+
+
+
+
+# ============================================================================
+# LOSS ФУНКЦИИ
+# ============================================================================
+
+
+def compute_loss(
+    logits: torch.Tensor,
+    input_ids: torch.Tensor,
+    context_lengths: torch.Tensor,
+    attention_mask: torch.Tensor,
+) -> dict:
+    """Вычисление loss для авторегрессионной модели."""
+    batch_size, seq_len, vocab_size = logits.shape
+
+    shift_logits = logits[:, :-1, :].contiguous()
+    shift_labels = input_ids[:, 1:].contiguous()
+    shift_mask = attention_mask[:, 1:].contiguous()
+
+    target_mask = torch.zeros_like(shift_labels, dtype=torch.bool)
+    for i in range(batch_size):
+        ctx_len = context_lengths[i].item()
+        target_mask[i, ctx_len - 1 :] = True
+
+    final_mask = target_mask & shift_mask.bool()
+
+    if final_mask.sum() > 0:
+        loss = F.cross_entropy(
+            shift_logits[final_mask], shift_labels[final_mask], reduction="mean"
+        )
+    else:
+        loss = torch.tensor(0.0, device=logits.device)
+
+    return {"loss": loss}
+
+
+def _pythia_forward_loss(
+    model: nn.Module,
+    batch: dict,
+    cfg: DictConfig,
+    accelerator: Accelerator,
+) -> dict:
+    """Forward + loss for a plain HF causal LM (attention_mask= kwarg, .logits)."""
+    input_ids = batch["input_ids"]
+    attention_mask = batch["attention_mask"]
+    context_lengths = batch["context_lengths"]
+    output = model(input_ids, attention_mask=attention_mask)
+    return compute_loss(output.logits, input_ids, context_lengths, attention_mask)
+
+
+# ============================================================================
+# PARAMETER GROUPING
+# ============================================================================
+
+
+def group_params(model: nn.Module, weight_decay: float) -> list[dict]:
+    """Группировка параметров для optimizer."""
+    decay_params = []
+    no_decay_params = []
+    
+    for name, param in model.named_parameters():
+        if not param.requires_grad:
+            continue
+            
+        if "bias" in name or "LayerNorm" in name or "layernorm" in name:
+            no_decay_params.append(param)
+        else:
+            decay_params.append(param)
+    
+    return [
+        {"params": decay_params, "weight_decay": weight_decay},
+        {"params": no_decay_params, "weight_decay": 0.0},
+    ]
+
+
+
+
+# ============================================================================
+# TRAINING LOOP
+# ============================================================================
+
+
+def train_epoch(
+    model: nn.Module,
+    dataloader: DataLoader,
+    optimizer: torch.optim.Optimizer,
+    scheduler,
+    cfg: DictConfig,
+    epoch: int,
+    global_step: int,
+    accelerator: Accelerator,
+    val_dataloader: DataLoader | None = None,
+    best_val_loss: float = float("inf"),
+) -> tuple[int, float]:
+    """Один epoch тренировки. Возвращает (global_step, best_val_loss)."""
+    model.train()
+
+    loss_meter = AverageMeter()
+
+    optimizer.zero_grad()
+    accumulated_loss = 0.0
+    accumulated_steps = 0
+
+    epoch_start_time = time.time()
+    step_start_time = time.time()
+
+    for batch_idx, batch in enumerate(dataloader):
+        input_ids = batch["input_ids"]
+        attention_mask = batch["attention_mask"]
+        context_lengths = batch["context_lengths"]
+
+        with accelerator.autocast():
+            output = model(input_ids, attention_mask=attention_mask)
+            logits = output.logits
+            loss_dict = compute_loss(
+                logits, input_ids, context_lengths, attention_mask
+            )
+
+        loss = loss_dict["loss"] / cfg.training.gradient_accumulation_steps
+        accelerator.backward(loss)
+
+        accumulated_loss += loss_dict["loss"].item()
+        accumulated_steps += 1
+
+        if accumulated_steps == cfg.training.gradient_accumulation_steps:
+            if cfg.training.max_grad_norm > 0:
+                accelerator.clip_grad_norm_(
+                    model.parameters(), cfg.training.max_grad_norm
+                )
+
+            optimizer.step()
+            scheduler.step()
+            optimizer.zero_grad()
+
+            avg_loss = accumulated_loss / cfg.training.gradient_accumulation_steps
+            loss_meter.update(avg_loss)
+
+            global_step += 1
+
+            if global_step % cfg.logging.log_interval == 0:
+                step_time = time.time() - step_start_time
+                current_lr = scheduler.get_last_lr()[0]
+
+                metrics = {
+                    "train/loss": loss_meter.val,
+                    "train/loss_avg": loss_meter.avg,
+                    "train/lr": current_lr,
+                    "train/epoch": epoch,
+                    "train/step_time": step_time / cfg.logging.log_interval,
+                }
+
+                log_metrics(metrics, step=global_step)
+
+                log_message(
+                    f"Epoch {epoch} | Step {global_step} | "
+                    f"Loss: {loss_meter.avg:.4f} | "
+                    f"LR: {current_lr:.2e}",
+                    cfg,
+                    accelerator,
+                )
+
+                step_start_time = time.time()
+
+            if (
+                cfg.logging.save_interval > 0
+                and global_step % cfg.logging.save_interval == 0
+            ):
+                save_checkpoint(
+                    model, optimizer, scheduler, global_step, epoch, cfg, accelerator
+                )
+
+            eval_interval = cfg.logging.get("eval_interval", 0)
+            if (
+                eval_interval > 0
+                and val_dataloader is not None
+                and global_step % eval_interval == 0
+            ):
+                val_metrics = run_validation(
+                    model=model,
+                    dataloader=val_dataloader,
+                    cfg=cfg,
+                    global_step=global_step,
+                    accelerator=accelerator,
+                    forward_loss_fn=_pythia_forward_loss,
+                )
+
+                if val_metrics["val/loss"] < best_val_loss:
+                    best_val_loss = val_metrics["val/loss"]
+                    if accelerator.is_main_process:
+                        best_model_path = Path(cfg.paths.output_dir) / "model_best.pt"
+                        unwrapped_model = accelerator.unwrap_model(model)
+                        torch.save(unwrapped_model.state_dict(), best_model_path)
+                        log_message(
+                            f"New best model saved! Val loss: {best_val_loss:.4f}", 
+                            cfg, 
+                            accelerator
+                        )
+
+                        log_metrics(
+                            {
+                                "best/val_loss": best_val_loss,
+                                "best/val_perplexity": val_metrics["val/perplexity"],
+                                "best/step": global_step,
+                            },
+                            step=global_step,
+                        )
+
+                model.train()
+
+            accumulated_loss = 0.0
+            accumulated_steps = 0
+
+    epoch_time = time.time() - epoch_start_time
+    
+    log_message(
+        f"Epoch {epoch} completed in {epoch_time:.2f}s | "
+        f"Loss: {loss_meter.avg:.4f}",
+        cfg,
+        accelerator,
+    )
+
+    log_metrics({
+        "epoch/loss": loss_meter.avg,
+        "epoch/time": epoch_time,
+    })
+
+    return global_step, best_val_loss
+
+
+# ============================================================================
+# MAIN
+# ============================================================================
+
+
+@hydra.main(version_base=None, config_path="configs", config_name="config")
+def main(cfg: DictConfig):
+    """Главная функция тренировки с поддержкой DDP через Accelerate."""
+
+    # === Performance: Enable TF32 for faster matmuls on Ampere+ GPUs ===
+    torch.set_float32_matmul_precision('high')
+
+    # === Accelerator Setup ===
+    mixed_precision = "bf16" if cfg.training.use_amp else "no"
+    
+    accelerator = Accelerator(
+        mixed_precision=mixed_precision,
+        gradient_accumulation_steps=cfg.training.gradient_accumulation_steps,
+    )
+
+    # === Setup ===
+    accelerate_set_seed(cfg.seed)
+    
+    if cfg.paths.output_dir is None:
+        cfg.paths.output_dir = HydraConfig.get().runtime.output_dir
+
+    OmegaConf.resolve(cfg)
+
+    log_message(f"CUDA_VISIBLE_DEVICES: {os.environ.get('CUDA_VISIBLE_DEVICES', 'not set')}", cfg, accelerator)
+    log_message(f"Number of processes: {accelerator.num_processes}", cfg, accelerator)
+    log_message(f"Process index: {accelerator.process_index}", cfg, accelerator)
+    log_message(f"Mixed precision: {mixed_precision}", cfg, accelerator)
+    
+    log_message("=" * 60, cfg, accelerator)
+    log_message("Pythia Training Pipeline (Hydra + Trackio + Accelerate)", cfg, accelerator)
+    log_message("=" * 60, cfg, accelerator)
+    log_message(f"Config:\n{OmegaConf.to_yaml(cfg)}", cfg, accelerator)
+
+    # === Trackio Init ===
+    init_tracking(cfg, accelerator)
+
+    # === Tokenizer ===
+    log_message("Initializing tokenizer...", cfg, accelerator)
+    tokenizer = AutoTokenizer.from_pretrained(cfg.model.name)
+    
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+        tokenizer.pad_token_id = tokenizer.eos_token_id
+
+    # === Model ===
+    log_message("Loading model...", cfg, accelerator)
+    
+    # Flash Attention 2
+    torch_dtype = torch.bfloat16 if cfg.training.use_amp else torch.float32
+    
+    if cfg.model.checkpoint_path:
+        model = AutoModelForCausalLM.from_pretrained(
+            cfg.model.name,
+            attn_implementation="flash_attention_2",
+            torch_dtype=torch_dtype,
+        )
+        checkpoint = torch.load(cfg.model.checkpoint_path, map_location="cpu")
+        model.load_state_dict(checkpoint["model_state_dict"] if "model_state_dict" in checkpoint else checkpoint)
+        log_message(f"Loaded checkpoint: {cfg.model.checkpoint_path}", cfg, accelerator)
+    elif cfg.model.from_scratch:
+        config = AutoConfig.from_pretrained(cfg.model.name)
+        config._attn_implementation = "flash_attention_2"
+        model = AutoModelForCausalLM.from_config(config, torch_dtype=torch_dtype)
+        log_message(f"Initialized from scratch: {cfg.model.name}", cfg, accelerator)
+    else:
+        model = AutoModelForCausalLM.from_pretrained(
+            cfg.model.name,
+            attn_implementation="flash_attention_2",
+            torch_dtype=torch_dtype,
+        )
+        log_message(f"Loaded pretrained: {cfg.model.name}", cfg, accelerator)
+
+    model.train()
+
+    # Log model info
+    total_params = sum(p.numel() for p in model.parameters())
+    trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
+    log_message(f"Total params: {total_params:,}", cfg, accelerator)
+    log_message(f"Trainable params: {trainable_params:,}", cfg, accelerator)
+
+    # === Data ===
+    log_message("Creating dataloaders...", cfg, accelerator)
+    dataloaders = create_dataloaders(cfg, tokenizer)
+
+    train_dataloader = dataloaders["train"]
+    val_dataloader = dataloaders.get("validation", None)
+
+    log_message(f"Train dataset size: {len(train_dataloader.dataset)}", cfg, accelerator)
+    log_message(f"Train batches per epoch (before DDP split): {len(train_dataloader)}", cfg, accelerator)
+
+    if val_dataloader:
+        log_message(f"Validation dataset size: {len(val_dataloader.dataset)}", cfg, accelerator)
+        log_message(f"Validation batches: {len(val_dataloader)}", cfg, accelerator)
+    else:
+        log_message("No validation dataset found", cfg, accelerator)
+
+    # === Optimizer ===
+    log_message("Creating optimizer...", cfg, accelerator)
+    param_groups = group_params(model, cfg.training.weight_decay)
+
+    optimizer = torch.optim.AdamW(
+        param_groups,
+        lr=cfg.training.lr,
+        betas=tuple(cfg.training.betas),
+        eps=cfg.training.eps,
+    )
+
+    # === Scheduler ===
+    steps_per_epoch = math.ceil(
+        len(train_dataloader) / accelerator.num_processes
+    )
+    total_steps = (
+        cfg.training.epochs
+        * steps_per_epoch
+        // cfg.training.gradient_accumulation_steps
+    )
+    scheduler = get_lr_scheduler(optimizer, cfg, total_steps)
+
+    log_message(
+        f"Total steps: {total_steps}, Steps per epoch: {steps_per_epoch}", 
+        cfg, 
+        accelerator
+    )
+
+    # === Accelerate Prepare ===
+    log_message("Preparing model, optimizer, and dataloaders with Accelerate...", cfg, accelerator)
+    
+    if val_dataloader is not None:
+        model, optimizer, train_dataloader, val_dataloader, scheduler = accelerator.prepare(
+            model, optimizer, train_dataloader, val_dataloader, scheduler
+        )
+    else:
+        model, optimizer, train_dataloader, scheduler = accelerator.prepare(
+            model, optimizer, train_dataloader, scheduler
+        )
+
+    log_message(f"Train batches per epoch (after DDP split): {len(train_dataloader)}", cfg, accelerator)
+
+    # === Resume ===
+    global_step = 0
+    start_epoch = 1
+
+    if cfg.training.resume and cfg.training.resume_checkpoint:
+        global_step, start_epoch = load_checkpoint(
+            model, optimizer, scheduler, cfg.training.resume_checkpoint, cfg, accelerator
+        )
+        start_epoch += 1
+
+    # === Training Loop ===
+    log_message("Starting training...", cfg, accelerator)
+
+    best_val_loss = float("inf")
+
+    try:
+        for epoch in range(start_epoch, cfg.training.epochs + 1):
+            log_message(f"\n{'=' * 60}", cfg, accelerator)
+            log_message(f"EPOCH {epoch}/{cfg.training.epochs}", cfg, accelerator)
+            log_message(f"{'=' * 60}", cfg, accelerator)
+
+            global_step, best_val_loss = train_epoch(
+                model=model,
+                dataloader=train_dataloader,
+                optimizer=optimizer,
+                scheduler=scheduler,
+                cfg=cfg,
+                epoch=epoch,
+                global_step=global_step,
+                accelerator=accelerator,
+                val_dataloader=val_dataloader,
+                best_val_loss=best_val_loss,
+            )
+
+            if cfg.logging.save_every_epoch:
+                save_checkpoint(
+                    model, optimizer, scheduler, global_step, epoch, cfg, accelerator
+                )
+
+    except KeyboardInterrupt:
+        log_message("Training interrupted by user", cfg, accelerator)
+        save_checkpoint(model, optimizer, scheduler, global_step, epoch, cfg, accelerator)
+
+    # === Final Save ===
+    log_message("\nTraining completed!", cfg, accelerator)
+
+    if accelerator.is_main_process:
+        final_model_path = Path(cfg.paths.output_dir) / "model_final.pt"
+        unwrapped_model = accelerator.unwrap_model(model)
+        torch.save(unwrapped_model.state_dict(), final_model_path)
+        log_message(f"Final model: {final_model_path}", cfg, accelerator)
+
+    accelerator.wait_for_everyone()
+    finish_tracking()
+
+
+if __name__ == "__main__":
+    main()

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/files/config.yaml

@@ -0,0 +1,154 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    code_path: code/code_completion_exp/train_pythia/train.py
+    python_version: 3.12.0
+    cli_version: 0.24.0
+    framework: huggingface
+    huggingface_version: 4.57.6
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1777154164
+    t:
+      1:
+      - 1
+      - 11
+      - 49
+      - 50
+      - 51
+      - 71
+      - 105
+      2:
+      - 1
+      - 11
+      - 49
+      - 50
+      - 51
+      - 71
+      - 105
+      3:
+      - 2
+      - 13
+      - 16
+      - 37
+      - 42
+      - 61
+      4: 3.12.0
+      5: 0.24.0
+      6: 4.57.6
+      13: linux-x86_64
+    e:
+      iazobdn073n3heo7pq8hjqzxa01dq9xi:
+        os: Linux-5.4.0-176-generic-x86_64-with-glibc2.35
+        python: CPython 3.12.0
+        started_at: '2026-04-25T21:56:04.223508Z'
+        program: /workspace/byte-llms-code/code_completion_exp/train_pythia/train.py
+        code_path: code_completion_exp/train_pythia/train.py
+        code_path_local: train.py
+        git:
+          remote_url: https://github.com/naryst/byte-llms-code.git
+          commit: 0a7180b6ab9f63d2794494f09ec4918576d10fa2
+        email: nikita@local.ru
+        root: outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5
+        host: 7504e518d24a
+        executable: /venv/bytellm/bin/python
+        cpu_count: 64
+        cpu_count_logical: 128
+        gpu_type: NVIDIA H100 80GB HBM3
+        gpu_count: 4
+        disk:
+          /:
+            total: '265214230528'
+            used: '111477301248'
+        memory:
+          total: '1081679683584'
+        gpu_nvidia:
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-b60cdcab-2033-2009-41de-be646c953a20
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-9982b420-4520-4238-c378-ec5a46015474
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-e26ebaac-aaa6-3eed-17ab-a3dce303a76f
+        - name: NVIDIA H100 80GB HBM3
+          memory_total: '85520809984'
+          cuda_cores: 16896
+          architecture: Hopper
+          uuid: GPU-9dfc6dba-0be6-4a10-1027-336cc0e65134
+        cuda_version: '12.2'
+        writer_id: iazobdn073n3heo7pq8hjqzxa01dq9xi
+model:
+  desc: null
+  value:
+    name: EleutherAI/pythia-1.4b
+    checkpoint_path: null
+    from_scratch: false
+training:
+  desc: null
+  value:
+    epochs: 3
+    batch_size: 4
+    eval_batch_size: 12
+    gradient_accumulation_steps: 4
+    lr: 2.0e-05
+    weight_decay: 0.1
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    lr_scheduler: wsd
+    warmup_ratio: 0.1
+    decay_ratio: 0.2
+    warmup_steps: 100
+    min_lr_ratio: 0.1
+    max_grad_norm: 1.0
+    use_amp: true
+    resume: false
+    resume_checkpoint: null
+data:
+  desc: null
+  value:
+    path: /workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full
+    max_context_len: 4096
+    max_target_len: 256
+    num_workers: 4
+    pin_memory: true
+    max_train_samples: null
+    max_val_samples: null
+logging:
+  desc: null
+  value:
+    log_interval: 10
+    save_interval: 6000
+    eval_interval: 1000
+    save_every_epoch: true
+tracking:
+  desc: null
+  value:
+    enabled: true
+    backend: wandb
+    project: code-completion_pythia-1.4b-rerun
+    run_name: pythia_1_4b_lr_2e-5
+    entity: null
+    base_url: https://wandb.platun0v.ru
+    local_dir: outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5
+paths:
+  desc: null
+  value:
+    output_dir: outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5
+seed:
+  desc: null
+  value: 42
+device:
+  desc: null
+  value: cuda

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/files/requirements.txt

@@ -0,0 +1,245 @@
+setuptools==78.1.1
+wheel==0.45.1
+pip==25.2
+webencodings==0.5.1
+triton==3.2.0
+pytz==2025.2
+pydub==0.25.1
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-ml-py==13.590.48
+nvidia-cusparselt-cu12==0.6.2
+mpmath==1.3.0
+ipython-genutils==0.2.0
+fastjsonschema==2.21.2
+brotli==1.2.0
+antlr4-python3-runtime==4.9.3
+xxhash==3.6.0
+widgetsnbextension==4.0.14
+websocket-client==1.9.0
+webcolors==24.11.1
+wcwidth==0.2.14
+urllib3==2.5.0
+uri-template==1.3.0
+tzdata==2025.2
+typing_extensions==4.15.0
+types-python-dateutil==2.9.0.20251008
+traitlets==5.14.3
+tqdm==4.67.1
+tornado==6.5.2
+tomlkit==0.13.3
+tinycss2==1.4.0
+tabulate==0.9.0
+sympy==1.13.1
+soupsieve==2.8
+sniffio==1.3.1
+smmap==5.0.2
+six==1.17.0
+shellingham==1.5.4
+Send2Trash==1.8.3
+semantic-version==2.10.0
+safetensors==0.6.2
+rpds-py==0.27.1
+rfc3986-validator==0.1.1
+regex==2025.9.18
+pyzmq==27.1.0
+PyYAML==6.0.3
+python-multipart==0.0.22
+python-json-logger==4.0.0
+python-dotenv==1.2.1
+pyparsing==3.2.5
+PyJWT==2.8.0
+Pygments==2.19.2
+pycparser==2.23
+pyarrow==22.0.0
+psutil==7.1.0
+protobuf==6.33.4
+propcache==0.4.1
+prometheus_client==0.23.1
+portalocker==3.2.0
+platformdirs==4.5.0
+pillow==11.3.0
+pexpect==4.9.0
+pathspec==1.0.4
+parso==0.8.5
+pandocfilters==1.5.1
+packaging==25.0
+orjson==3.11.6
+opt_einsum==3.4.0
+nvidia-nvtx-cu12==12.4.127
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nccl-cu12==2.21.5
+nvidia-curand-cu12==10.3.5.147
+nvidia-cufile-cu12==1.13.1.3
+nvidia-cufft-cu12==11.2.1.3
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cublas-cu12==12.4.5.8
+numpy==2.3.3
+ninja==1.13.0
+networkx==3.5
+nest-asyncio==1.6.0
+narwhals==2.15.0
+mypy_extensions==1.1.0
+multidict==6.7.0
+mistune==3.1.4
+mdurl==0.1.2
+MarkupSafe==3.0.3
+lxml==6.0.2
+librt==0.8.0
+lark==1.3.0
+kiwisolver==1.4.9
+jupyterlab_widgets==3.0.15
+jupyterlab_pygments==0.3.0
+jsonpointer==3.0.0
+json5==0.12.1
+itsdangerous==2.2.0
+idna==3.10
+hf-xet==1.1.10
+h11==0.16.0
+groovy==0.1.2
+fsspec==2025.9.0
+frozenlist==1.8.0
+fqdn==1.5.1
+fonttools==4.60.1
+filelock==3.19.1
+ffmpy==1.0.0
+executing==2.2.1
+einops==0.8.1
+dill==0.4.0
+defusedxml==0.7.1
+decorator==5.2.1
+debugpy==1.8.17
+dacite==1.9.2
+cycler==0.12.1
+comm==0.2.3
+colorama==0.4.6
+click==8.3.1
+charset-normalizer==3.4.3
+certifi==2025.10.5
+bleach==6.2.0
+babel==2.17.0
+attrs==25.4.0
+async-lru==2.0.5
+asttokens==3.0.0
+annotated-types==0.7.0
+annotated-doc==0.0.4
+aiohappyeyeballs==2.6.1
+aiofiles==24.1.0
+yarl==1.22.0
+uvicorn==0.40.0
+typing-inspection==0.4.2
+terminado==0.18.1
+stack-data==0.6.3
+sentry-sdk==2.50.0
+scipy==1.17.0
+sacrebleu==2.6.0
+rfc3987-syntax==1.1.0
+rfc3339-validator==0.1.4
+requests==2.32.5
+reportlab==4.4.9
+referencing==0.36.2
+python-dateutil==2.9.0.post0
+pydantic_core==2.41.5
+prompt_toolkit==3.0.52
+plotly==6.5.2
+pathlib2==2.3.7.post1
+orderedmultidict==1.0.2
+optree==0.17.0
+omegaconf==2.3.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+mypy==1.19.1
+multiprocess==0.70.16
+matplotlib-inline==0.1.7
+markdown-it-py==4.0.0
+jupyter_core==5.8.1
+Jinja2==3.1.6
+jedi==0.19.2
+ipython_pygments_lexers==1.1.1
+httpcore==1.0.9
+gitdb==4.0.12
+ftfy==6.3.1
+contourpy==1.3.3
+cffi==2.0.0
+beautifulsoup4==4.14.2
+anyio==4.11.0
+aiosignal==1.4.0
+starlette==0.50.0
+rich==14.2.0
+pydantic==2.12.5
+pandas==2.3.3
+nvidia-cusolver-cu12==11.6.1.9
+matplotlib==3.10.7
+jupyter_server_terminals==0.5.3
+jupyter_client==8.6.3
+jsonschema-specifications==2025.9.1
+ipython==9.6.0
+hydra-core==1.3.2
+huggingface-hub==0.35.3
+httpx==0.28.1
+GitPython==3.1.46
+furl==2.1.4
+cryptography==46.0.4
+arrow==1.3.0
+argon2-cffi-bindings==25.1.0
+aiohttp==3.13.1
+wandb==0.24.0
+typer==0.21.1
+torch==2.6.0
+tokenizers==0.22.1
+seaborn==0.13.2
+safehttpx==0.1.7
+jsonschema==4.25.1
+joypy==0.2.6
+isoduration==20.11.0
+ipywidgets==8.1.7
+ipykernel==6.30.1
+gradio_client==2.0.3
+fastapi==0.128.0
+Authlib==1.6.6
+argon2-cffi==25.1.0
+transformers==4.57.6
+nbformat==5.10.4
+mlstm_kernels==2.0.2
+jupyter-console==6.6.3
+gradio==6.5.1
+datasets==4.3.0
+clearml==1.16.4
+accelerate==1.10.1
+xlstm==2.0.4
+nbclient==0.10.2
+jupyter-events==0.12.0
+trackio==0.15.0
+nbconvert==7.16.6
+jupyter_server==2.17.0
+notebook_shim==0.2.4
+jupyterlab_server==2.27.3
+jupyter-lsp==2.3.0
+nbclassic==1.3.3
+jupyterlab==4.4.9
+notebook==7.4.7
+jupyter_contrib_core==0.4.2
+jupyter==1.1.1
+jupyter_nbextensions_configurator==0.6.4
+causal-conv1d==1.5.0.post8
+flash_attn==2.7.4.post1
+mamba-ssm==2.2.4
+hnet==0.0.1
+autocommand==2.2.2
+backports.tarfile==1.2.0
+importlib_metadata==8.0.0
+inflect==7.3.1
+jaraco.collections==5.1.0
+jaraco.context==5.3.0
+jaraco.functools==4.0.1
+jaraco.text==3.12.1
+more-itertools==10.3.0
+packaging==24.2
+platformdirs==4.2.2
+tomli==2.0.1
+typeguard==4.3.0
+typing_extensions==4.12.2
+wheel==0.45.1
+zipp==3.19.2

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/files/wandb-metadata.json

@@ -0,0 +1 @@
+{"os": "Linux-5.4.0-176-generic-x86_64-with-glibc2.35", "python": "CPython 3.12.0", "started_at": "2026-04-25T21:56:04.223508Z", "program": "/workspace/byte-llms-code/code_completion_exp/train_pythia/train.py", "code_path": "code_completion_exp/train_pythia/train.py", "code_path_local": "train.py", "git": {"remote_url": "https://github.com/naryst/byte-llms-code.git", "commit": "0a7180b6ab9f63d2794494f09ec4918576d10fa2"}, "email": "nikita@local.ru", "root": "outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5", "host": "7504e518d24a", "executable": "/venv/bytellm/bin/python", "cpu_count": 64, "cpu_count_logical": 128, "gpu_type": "NVIDIA H100 80GB HBM3", "gpu_count": 4, "disk": {"/": {"total": "265214230528", "used": "111477301248"}}, "memory": {"total": "1081679683584"}, "gpu_nvidia": [{"name": "NVIDIA H100 80GB HBM3", "memory_total": "85520809984", "cuda_cores": 16896, "architecture": "Hopper", "uuid": "GPU-b60cdcab-2033-2009-41de-be646c953a20"}, {"name": "NVIDIA H100 80GB HBM3", "memory_total": "85520809984", "cuda_cores": 16896, "architecture": "Hopper", "uuid": "GPU-9982b420-4520-4238-c378-ec5a46015474"}, {"name": "NVIDIA H100 80GB HBM3", "memory_total": "85520809984", "cuda_cores": 16896, "architecture": "Hopper", "uuid": "GPU-e26ebaac-aaa6-3eed-17ab-a3dce303a76f"}, {"name": "NVIDIA H100 80GB HBM3", "memory_total": "85520809984", "cuda_cores": 16896, "architecture": "Hopper", "uuid": "GPU-9dfc6dba-0be6-4a10-1027-336cc0e65134"}], "cuda_version": "12.2", "writer_id": "iazobdn073n3heo7pq8hjqzxa01dq9xi"}

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime": 13701, "_timestamp": 1777167844.718428, "_step": 29660, "train/loss": 0.6748308092355728, "train/loss_avg": 0.7016983942730095, "train/lr": 2.0000000000000003e-06, "train/epoch": 3, "train/step_time": 0.3627600908279419, "val/loss": 0.9848005849860421, "val/time": 92.58129596710205, "best/val_perplexity": 2.6676781909665626, "best/step": 19000, "val/perplexity": 2.712564749615551, "best/val_loss": 0.9681412068259936, "epoch/loss": 0.7017153006575418, "epoch/time": 4552.06911277771}

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2026-04-25T21:56:04.307874507Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmptd7__oni/port-185429.txt","pid":185429,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-04-25T21:56:04.308305608Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":185429}
+{"time":"2026-04-25T21:56:04.30830261Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-185429-185491-3756380617/socket","Net":"unix"}}
+{"time":"2026-04-25T21:56:04.495821383Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-04-25T21:56:04.515259356Z","level":"INFO","msg":"handleInformInit: received","streamId":"jhqe4qjw","id":"1(@)"}
+{"time":"2026-04-25T21:56:04.914894772Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"jhqe4qjw","id":"1(@)"}
+{"time":"2026-04-26T01:44:27.124695645Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"jhqe4qjw","id":"1(@)"}
+{"time":"2026-04-26T01:44:27.125165614Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"jhqe4qjw","id":"1(@)"}
+{"time":"2026-04-26T01:44:27.144975551Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-04-26T01:44:27.145027858Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-04-26T01:44:27.145044227Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-04-26T01:44:27.145084129Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-04-26T01:44:27.145088296Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-04-26T01:44:27.145089567Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-04-26T01:44:27.145199385Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-185429-185491-3756380617/socket","Net":"unix"}}
+{"time":"2026-04-26T01:44:27.145231896Z","level":"INFO","msg":"server is closed"}

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2026-04-25T21:56:04.515355828Z","level":"INFO","msg":"stream: starting","core version":"0.24.0"}
+{"time":"2026-04-25T21:56:04.914734923Z","level":"INFO","msg":"stream: created new stream","id":"jhqe4qjw"}
+{"time":"2026-04-25T21:56:04.91479529Z","level":"INFO","msg":"handler: started","stream_id":"jhqe4qjw"}
+{"time":"2026-04-25T21:56:04.914888584Z","level":"INFO","msg":"stream: started","id":"jhqe4qjw"}
+{"time":"2026-04-25T21:56:04.914893063Z","level":"INFO","msg":"writer: started","stream_id":"jhqe4qjw"}
+{"time":"2026-04-25T21:56:04.914903421Z","level":"INFO","msg":"sender: started","stream_id":"jhqe4qjw"}
+{"time":"2026-04-25T21:56:05.060789967Z","level":"ERROR","msg":"git repo not found","error":"repository does not exist"}
+{"time":"2026-04-25T23:20:15.280635585Z","level":"ERROR","msg":"api: HTTP error","status":403,"method":"POST","url":"https://wandb.platun0v.ru/files/nikita/code-completion_pythia-1.4b-rerun/jhqe4qjw/file_stream"}
+{"time":"2026-04-25T23:20:15.280696408Z","level":"ERROR+4","msg":"filestream: fatal error: filestream: failed to upload: 403 Forbidden url=https://wandb.platun0v.ru/files/nikita/code-completion_pythia-1.4b-rerun/jhqe4qjw/file_stream: "}
+{"time":"2026-04-26T01:44:27.121098166Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-04-26T01:44:27.121923843Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2026-04-26T01:44:27.124737334Z","level":"INFO","msg":"stream: closing","id":"jhqe4qjw"}
+{"time":"2026-04-26T01:44:27.124749132Z","level":"INFO","msg":"handler: closed","stream_id":"jhqe4qjw"}
+{"time":"2026-04-26T01:44:27.124842763Z","level":"INFO","msg":"sender: closed","stream_id":"jhqe4qjw"}
+{"time":"2026-04-26T01:44:27.124848143Z","level":"INFO","msg":"stream: closed","id":"jhqe4qjw"}

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/logs/debug.log

@@ -0,0 +1,24 @@
+2026-04-25 21:56:04,224 INFO    MainThread:185429 [wandb_setup.py:_flush():81] Current SDK version is 0.24.0
+2026-04-25 21:56:04,224 INFO    MainThread:185429 [wandb_setup.py:_flush():81] Configure stats pid to 185429
+2026-04-25 21:56:04,224 INFO    MainThread:185429 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-04-25 21:56:04,225 INFO    MainThread:185429 [wandb_init.py:setup_run_log_directory():717] Logging user logs to outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/logs/debug.log
+2026-04-25 21:56:04,225 INFO    MainThread:185429 [wandb_init.py:setup_run_log_directory():718] Logging internal logs to outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/logs/debug-internal.log
+2026-04-25 21:56:04,225 INFO    MainThread:185429 [wandb_init.py:init():844] calling init triggers
+2026-04-25 21:56:04,225 INFO    MainThread:185429 [wandb_init.py:init():849] wandb.init called with sweep_config: {}
+config: {'model': {'name': 'EleutherAI/pythia-1.4b', 'checkpoint_path': None, 'from_scratch': False}, 'training': {'epochs': 3, 'batch_size': 4, 'eval_batch_size': 12, 'gradient_accumulation_steps': 4, 'lr': 2e-05, 'weight_decay': 0.1, 'betas': [0.9, 0.95], 'eps': 1e-08, 'lr_scheduler': 'wsd', 'warmup_ratio': 0.1, 'decay_ratio': 0.2, 'warmup_steps': 100, 'min_lr_ratio': 0.1, 'max_grad_norm': 1.0, 'use_amp': True, 'resume': False, 'resume_checkpoint': None}, 'data': {'path': '/workspace/byte-llms-code/code_completion_exp/datasets/data_V4_full', 'max_context_len': 4096, 'max_target_len': 256, 'num_workers': 4, 'pin_memory': True, 'max_train_samples': None, 'max_val_samples': None}, 'logging': {'log_interval': 10, 'save_interval': 6000, 'eval_interval': 1000, 'save_every_epoch': True}, 'tracking': {'enabled': True, 'backend': 'wandb', 'project': 'code-completion_pythia-1.4b-rerun', 'run_name': 'pythia_1_4b_lr_2e-5', 'entity': None, 'base_url': 'https://wandb.platun0v.ru', 'local_dir': 'outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5'}, 'paths': {'output_dir': 'outputs/pythia_1_4b_rerun/pythia_1_4b_lr_2e-5'}, 'seed': 42, 'device': 'cuda', '_wandb': {'code_path': 'code/code_completion_exp/train_pythia/train.py'}}
+2026-04-25 21:56:04,225 INFO    MainThread:185429 [wandb_init.py:init():892] starting backend
+2026-04-25 21:56:04,496 INFO    MainThread:185429 [wandb_init.py:init():895] sending inform_init request
+2026-04-25 21:56:04,514 INFO    MainThread:185429 [wandb_init.py:init():903] backend started and connected
+2026-04-25 21:56:04,517 INFO    MainThread:185429 [wandb_init.py:init():973] updated telemetry
+2026-04-25 21:56:04,531 INFO    MainThread:185429 [wandb_init.py:init():997] communicating run to backend with 90.0 second timeout
+2026-04-25 21:56:05,060 INFO    MainThread:185429 [wandb_init.py:init():1044] starting run threads in backend
+2026-04-25 21:56:05,217 INFO    MainThread:185429 [wandb_run.py:_console_start():2529] atexit reg
+2026-04-25 21:56:05,217 INFO    MainThread:185429 [wandb_run.py:_redirect():2377] redirect: wrap_raw
+2026-04-25 21:56:05,217 INFO    MainThread:185429 [wandb_run.py:_redirect():2446] Wrapping output streams.
+2026-04-25 21:56:05,217 INFO    MainThread:185429 [wandb_run.py:_redirect():2469] Redirects installed.
+2026-04-25 21:56:05,220 INFO    MainThread:185429 [wandb_init.py:init():1084] run started, returning control to user process
+2026-04-26 01:44:26,160 INFO    MainThread:185429 [wandb_run.py:_finish():2295] finishing run nikita/code-completion_pythia-1.4b-rerun/jhqe4qjw
+2026-04-26 01:44:26,160 INFO    MainThread:185429 [wandb_run.py:_atexit_cleanup():2494] got exitcode: 0
+2026-04-26 01:44:26,161 INFO    MainThread:185429 [wandb_run.py:_restore():2476] restore
+2026-04-26 01:44:26,161 INFO    MainThread:185429 [wandb_run.py:_restore():2482] restore done
+2026-04-26 01:44:27,124 INFO    MainThread:185429 [wandb_run.py:_footer_sync_info():3870] logging synced files

pythia_1_4b_rerun/pythia_1_4b_lr_2e-5/wandb/run-20260425_215604-jhqe4qjw/run-jhqe4qjw.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:547dc16fb780cece720bb77317513532a4d9564e9b81b6d1445bbd19e2123eff
+size 5773984