Copying trained fw57M-tied model

llm/fw57M-tied/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

llm/fw57M-tied/README.md

@@ -0,0 +1,90 @@
+---
+{}
+---
+## Experiment Configuration
+```yaml
+callbacks:
+  grad_accum:
+    _target_: src.callbacks.gradient_accumulation.GradientAccumulationScheduler
+    scheduling:
+      0: 4
+  grad_norm:
+    _target_: src.callbacks.grad_norm.GradNorm
+    check_clipping: false
+    group_separator: /
+    histogram_freq: null
+    log_weight_distribution: false
+    norm_type: 2
+    only_total: true
+  lr_monitor:
+    _target_: src.callbacks.lr_monitor.SimpleLearningRateMonitor
+  model_checkpoint:
+    _target_: src.callbacks.model_checkpoint.ModelCheckpoint
+    dirpath: .checkpoints
+    enable_version_counter: false
+    every_n_train_steps: 2000
+    filename: '{step}'
+    save_initial_checkpoint: true
+    save_last: link
+    save_top_k: -1
+    verbose: true
+  speed_monitor:
+    _target_: src.callbacks.speed_monitor.SpeedMonitor
+data:
+  batch_size: 32
+  drop_last: false
+  eval_batch_size: 128
+  multiprocessing_context: null
+  num_workers: 12
+  persistent_workers: false
+  pin_memory: true
+  prefetch_factor: 2
+  shuffle: true
+dataset: finewebedu-20B
+loggers:
+  tensorboard:
+    _target_: src.trainer.TensorBoardLogger
+    name: ''
+    save_dir: ./
+    version: null
+model: fw57M-tied
+optim:
+  lr: 0.0006
+  num_warmup_steps: 2000
+  optim_kwargs:
+    betas:
+    - 0.9
+    - 0.95
+    eps: 1.0e-08
+    fused: true
+  optim_name: adamw
+  scheduler_kwargs:
+    min_lr_ratio: 0.01
+    num_decay_steps: 4000
+    num_stable_steps: 44000
+  scheduler_name: warmup_stable_decay
+  weight_decay: 0.01
+out_parent_folder: model_train
+pwd: /home/zg258/projects/infotokenization
+resume_from_checkpoint: .checkpoints/last.ckpt
+run_folder: .
+save_initial_checkpoint: true
+seed: 42
+tok_name: bytelevel
+torch_compile: true
+train_data_path: /home/zg258/projects/infotokenization/data/finewebedu-20B/bytelevel-subset/train
+trainer:
+  accelerator: gpu
+  deterministic: false
+  devices: 1
+  enable_progress_bar: true
+  fast_dev_run: false
+  gradient_clip_algorithm: norm
+  gradient_clip_val: 1.0
+  limit_val_batches: 500
+  log_every_n_steps: 1
+  max_steps: 50000
+  precision: bf16-true
+  val_check_interval: 2000
+val_data_path: /home/zg258/projects/infotokenization/data/finewebedu-20B/bytelevel-subset/validation
+```

llm/fw57M-tied/config.json

@@ -0,0 +1,30 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": null,
+  "eos_token_id": 1,
+  "head_dim": 32,
+  "hidden_act": "silu",
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 24,
+  "num_hidden_layers": 6,
+  "num_key_value_heads": 24,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.1",
+  "use_cache": true,
+  "vocab_size": 258
+}

llm/fw57M-tied/generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.51.1"
+}

llm/fw57M-tied/hparams.yaml

@@ -0,0 +1,84 @@
+loggers:
+  tensorboard:
+    _target_: src.trainer.TensorBoardLogger
+    save_dir: ./
+    name: ''
+    version: null
+callbacks:
+  lr_monitor:
+    _target_: src.callbacks.lr_monitor.SimpleLearningRateMonitor
+  grad_norm:
+    _target_: src.callbacks.grad_norm.GradNorm
+    norm_type: 2
+    group_separator: /
+    histogram_freq: null
+    check_clipping: false
+    log_weight_distribution: false
+    only_total: true
+  speed_monitor:
+    _target_: src.callbacks.speed_monitor.SpeedMonitor
+  grad_accum:
+    _target_: src.callbacks.gradient_accumulation.GradientAccumulationScheduler
+    scheduling:
+      0: 4
+  model_checkpoint:
+    _target_: src.callbacks.model_checkpoint.ModelCheckpoint
+    dirpath: .checkpoints
+    filename: '{step}'
+    enable_version_counter: false
+    every_n_train_steps: 2000
+    save_top_k: -1
+    save_last: link
+    verbose: true
+    save_initial_checkpoint: true
+out_parent_folder: model_train
+tok_name: bytelevel
+run_folder: .
+dataset: finewebedu-20B
+pwd: /home/zg258/projects/infotokenization
+train_data_path: /home/zg258/projects/infotokenization/data/finewebedu-20B/bytelevel-subset/train
+val_data_path: /home/zg258/projects/infotokenization/data/finewebedu-20B/bytelevel-subset/validation
+model: fw57M-tied
+resume_from_checkpoint: .checkpoints/last.ckpt
+save_initial_checkpoint: true
+seed: 42
+torch_compile: true
+data:
+  batch_size: 32
+  eval_batch_size: 128
+  shuffle: true
+  drop_last: false
+  num_workers: 12
+  pin_memory: true
+  persistent_workers: false
+  prefetch_factor: 2
+  multiprocessing_context: null
+optim:
+  optim_name: adamw
+  lr: 0.0006
+  weight_decay: 0.01
+  optim_kwargs:
+    fused: true
+    eps: 1.0e-08
+    betas:
+    - 0.9
+    - 0.95
+  scheduler_name: warmup_stable_decay
+  num_warmup_steps: 2000
+  scheduler_kwargs:
+    num_stable_steps: 44000
+    num_decay_steps: 4000
+    min_lr_ratio: 0.01
+trainer:
+  accelerator: gpu
+  devices: 1
+  precision: bf16-true
+  deterministic: false
+  log_every_n_steps: 1
+  enable_progress_bar: true
+  fast_dev_run: false
+  gradient_clip_val: 1.0
+  gradient_clip_algorithm: norm
+  val_check_interval: 2000
+  max_steps: 50000
+  limit_val_batches: 500

llm/fw57M-tied/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1c5989b2eb341dde8e68fbf142f4f53a66c40876344c9e2571331e98edb1bc3
+size 113668624

llm/fw57M-tied/tb_logs.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80adcbac1f2bb5ab25365d18ff532c89602efb8bdd02e03f6f5b1747f6ff6eee
+size 2853290