Delete llm/fw57M-tied/hparams.yaml

llm/fw57M-tied/hparams.yaml

@@ -1,84 +0,0 @@
-loggers:
-  tensorboard:
-    _target_: src.trainer.TensorBoardLogger
-    save_dir: ./
-    name: ''
-    version: null
-callbacks:
-  lr_monitor:
-    _target_: src.callbacks.lr_monitor.SimpleLearningRateMonitor
-  grad_norm:
-    _target_: src.callbacks.grad_norm.GradNorm
-    norm_type: 2
-    group_separator: /
-    histogram_freq: null
-    check_clipping: false
-    log_weight_distribution: false
-    only_total: true
-  speed_monitor:
-    _target_: src.callbacks.speed_monitor.SpeedMonitor
-  grad_accum:
-    _target_: src.callbacks.gradient_accumulation.GradientAccumulationScheduler
-    scheduling:
-      0: 4
-  model_checkpoint:
-    _target_: src.callbacks.model_checkpoint.ModelCheckpoint
-    dirpath: .checkpoints
-    filename: '{step}'
-    enable_version_counter: false
-    every_n_train_steps: 2000
-    save_top_k: -1
-    save_last: link
-    verbose: true
-    save_initial_checkpoint: true
-out_parent_folder: model_train
-tok_name: bytelevel
-run_folder: .
-dataset: finewebedu-20B
-pwd: /home/zg258/projects/infotokenization
-train_data_path: /home/zg258/projects/infotokenization/data/finewebedu-20B/bytelevel-subset/train
-val_data_path: /home/zg258/projects/infotokenization/data/finewebedu-20B/bytelevel-subset/validation
-model: fw57M-tied
-resume_from_checkpoint: .checkpoints/last.ckpt
-save_initial_checkpoint: true
-seed: 42
-torch_compile: true
-data:
-  batch_size: 32
-  eval_batch_size: 128
-  shuffle: true
-  drop_last: false
-  num_workers: 12
-  pin_memory: true
-  persistent_workers: false
-  prefetch_factor: 2
-  multiprocessing_context: null
-optim:
-  optim_name: adamw
-  lr: 0.0006
-  weight_decay: 0.01
-  optim_kwargs:
-    fused: true
-    eps: 1.0e-08
-    betas:
-    - 0.9
-    - 0.95
-  scheduler_name: warmup_stable_decay
-  num_warmup_steps: 2000
-  scheduler_kwargs:
-    num_stable_steps: 44000
-    num_decay_steps: 4000
-    min_lr_ratio: 0.01
-trainer:
-  accelerator: gpu
-  devices: 1
-  precision: bf16-true
-  deterministic: false
-  log_every_n_steps: 1
-  enable_progress_bar: true
-  fast_dev_run: false
-  gradient_clip_val: 1.0
-  gradient_clip_algorithm: norm
-  val_check_interval: 2000
-  max_steps: 50000
-  limit_val_batches: 500