Upload folder using huggingface_hub

best_hydra_mark-v1.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:194d919309bcf7e7cd62e9a76163e9bee2b8c72f9fff291fd28abf1c301698bb
+size 372899530

best_hydra_mark-v2.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2398c74a45047ec7b1fc7d3558e6cddbd66223cc9457141cb3c6848e5b5e4b75
+size 240007286

best_hydra_mark.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3e5ff83d9ea380a69098ca0fafe0a750d81bbf55d70542e884fc6553bd01a13
+size 266718065

lightning_logs/version_0/events.out.tfevents.1775832652.b85934d9bf78.22225.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eae8369e0d04e06207668fa9fa45c43963aed930794930af70923980d403c115
+size 21499

lightning_logs/version_0/hparams.yaml

@@ -0,0 +1,89 @@
+batch_size: 100
+epochs: 3
+stage: 1
+unfrozen_ratio: 0.3
+ckpt_weights_only: false
+checkpoint_dir: ./checkpoints/hydra_mark
+train_data_dir: ./data/train_shards1
+val_data_dir: ./data/val_shards
+weights_path: ./models/hydra_dct_mark.pt
+shuffle: true
+use_early_stopping: false
+max_patience_counter: 5
+min_delta: 0.01
+use_gradient_clipping: true
+gradient_clipping_norm: 1.0
+pad_length: 4096
+learning_rate_mark: 0.0006
+learning_rate_hydra: 3.0e-05
+learning_rate_cls: 0.0001
+no_cache: false
+num_workers: 8
+matmul_precision: high
+multi_shot: false
+intervals: 3
+is_prenorm: false
+accumulate_grad_batches: 2
+cart: true
+cart_p: 0.45
+cart_scale: 1.0
+distillation: false
+lr_scheduler:
+  type: cosine
+  warmup_steps: 850
+  total_steps: 16000
+  min_lr_ratio: 0.1
+  polynomial:
+    end_lr_ratio: 0.0
+    power: 1.0
+  plateau:
+    factor: 0.5
+    patience: 3
+    min_lr: 1.0e-06
+trainer:
+  accelerator: gpu
+  devices: -1
+  check_val_every_n_epoch: null
+  num_sanity_val_steps: 0
+  accumulate_grad_batches: 1
+  precision: bf16-true
+  enable_checkpointing: true
+  default_root_dir: ./checkpoints/hydra_mark
+wandb:
+  project: hydra-training-dct
+  model_name: HydraForMaskedLM
+  watch_log: all
+  log_freq: 20
+hydra_config:
+  hidden_size: 768
+  vocab_size: 30522
+  type_vocab_size: 2
+  pad_token_id: 0
+  use_position_embeddings: false
+  max_position_embeddings: 4096
+  use_timestep_embeddings: true
+  layer_norm_eps: 1.0e-12
+  dropout: 0.0
+  max_timestep_embeddings: 1000
+  current_timestep: 0
+  d_state: 64
+  d_conv: 7
+  head_dim: 64
+  expand: 2
+  chunk_size: 256
+  is_prenorm: false
+  use_eff_compute: false
+  gradient_checkpointing: true
+  num_hidden_layers: 23
+  guider_hidden_layers: 12
+  device: cpu
+  pool_all: false
+  mark_kernel: dct
+  mark_ensemble: false
+  rank: 2
+  degree: 5
+  L_timepoints: 256
+  n_freqs: 8
+  mark_mlp_dim: 256
+  hidden_act: swish
+  initializer_range: 0.02

lightning_logs/version_1/events.out.tfevents.1776215594.b85934d9bf78.35032.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:911b07a57a9644699b4e416e6482bca429d96013e7a369e77f4d74dec6c563c3
+size 21887

lightning_logs/version_1/hparams.yaml

@@ -0,0 +1,89 @@
+batch_size: 100
+epochs: 5
+stage: 2
+unfrozen_ratio: 0.3
+ckpt_weights_only: true
+checkpoint_dir: ./checkpoints/hydra_mark
+train_data_dir: ./data/train_shards2
+val_data_dir: ./data/val_shards
+weights_path: ./models/hydra_dct_mark.pt
+shuffle: true
+use_early_stopping: false
+max_patience_counter: 5
+min_delta: 0.01
+use_gradient_clipping: true
+gradient_clipping_norm: 1.0
+pad_length: 4096
+learning_rate_mark: 0.0006
+learning_rate_hydra: 3.0e-05
+learning_rate_cls: 0.0001
+no_cache: false
+num_workers: 8
+matmul_precision: high
+multi_shot: false
+intervals: 3
+is_prenorm: false
+accumulate_grad_batches: 2
+cart: true
+cart_p: 0.45
+cart_scale: 1.0
+distillation: false
+lr_scheduler:
+  type: cosine
+  warmup_steps: 850
+  total_steps: 16000
+  min_lr_ratio: 0.1
+  polynomial:
+    end_lr_ratio: 0.0
+    power: 1.0
+  plateau:
+    factor: 0.5
+    patience: 3
+    min_lr: 1.0e-06
+trainer:
+  accelerator: gpu
+  devices: -1
+  check_val_every_n_epoch: null
+  num_sanity_val_steps: 0
+  accumulate_grad_batches: 1
+  precision: bf16-true
+  enable_checkpointing: true
+  default_root_dir: ./checkpoints/hydra_mark
+wandb:
+  project: hydra-training-dct
+  model_name: HydraForMaskedLM
+  watch_log: all
+  log_freq: 20
+hydra_config:
+  hidden_size: 768
+  vocab_size: 30522
+  type_vocab_size: 2
+  pad_token_id: 0
+  use_position_embeddings: false
+  max_position_embeddings: 4096
+  use_timestep_embeddings: true
+  layer_norm_eps: 1.0e-12
+  dropout: 0.0
+  max_timestep_embeddings: 1000
+  current_timestep: 0
+  d_state: 64
+  d_conv: 7
+  head_dim: 64
+  expand: 2
+  chunk_size: 256
+  is_prenorm: false
+  use_eff_compute: false
+  gradient_checkpointing: true
+  num_hidden_layers: 23
+  guider_hidden_layers: 12
+  device: cpu
+  pool_all: false
+  mark_kernel: dct
+  mark_ensemble: false
+  rank: 2
+  degree: 5
+  L_timepoints: 256
+  n_freqs: 8
+  mark_mlp_dim: 256
+  hidden_act: swish
+  initializer_range: 0.02

lightning_logs/version_2/events.out.tfevents.1776269785.b85934d9bf78.38980.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d31ed6ff2c5a4d35f296c42eb2cdac3c832aa51881fffdb05f16b43e23b4adef
+size 7387

lightning_logs/version_2/hparams.yaml

@@ -0,0 +1,89 @@
+batch_size: 100
+epochs: 3
+stage: 3
+unfrozen_ratio: 0.3
+ckpt_weights_only: true
+checkpoint_dir: ./checkpoints/hydra_mark
+train_data_dir: ./data/train_shards3
+val_data_dir: ./data/val_shards
+weights_path: ./models/hydra_dct_mark.pt
+shuffle: true
+use_early_stopping: false
+max_patience_counter: 5
+min_delta: 0.01
+use_gradient_clipping: true
+gradient_clipping_norm: 1.0
+pad_length: 4096
+learning_rate_mark: 0.0006
+learning_rate_hydra: 3.0e-05
+learning_rate_cls: 0.0001
+no_cache: false
+num_workers: 8
+matmul_precision: high
+multi_shot: false
+intervals: 3
+is_prenorm: false
+accumulate_grad_batches: 2
+cart: true
+cart_p: 0.45
+cart_scale: 1.0
+distillation: false
+lr_scheduler:
+  type: cosine
+  warmup_steps: 850
+  total_steps: 16000
+  min_lr_ratio: 0.1
+  polynomial:
+    end_lr_ratio: 0.0
+    power: 1.0
+  plateau:
+    factor: 0.5
+    patience: 3
+    min_lr: 1.0e-06
+trainer:
+  accelerator: gpu
+  devices: -1
+  check_val_every_n_epoch: null
+  num_sanity_val_steps: 0
+  accumulate_grad_batches: 1
+  precision: bf16-true
+  enable_checkpointing: true
+  default_root_dir: ./checkpoints/hydra_mark
+wandb:
+  project: hydra-training-dct
+  model_name: HydraForMaskedLM
+  watch_log: all
+  log_freq: 20
+hydra_config:
+  hidden_size: 768
+  vocab_size: 30522
+  type_vocab_size: 2
+  pad_token_id: 0
+  use_position_embeddings: false
+  max_position_embeddings: 4096
+  use_timestep_embeddings: true
+  layer_norm_eps: 1.0e-12
+  dropout: 0.0
+  max_timestep_embeddings: 1000
+  current_timestep: 0
+  d_state: 64
+  d_conv: 7
+  head_dim: 64
+  expand: 2
+  chunk_size: 256
+  is_prenorm: false
+  use_eff_compute: false
+  gradient_checkpointing: true
+  num_hidden_layers: 23
+  guider_hidden_layers: 12
+  device: cpu
+  pool_all: false
+  mark_kernel: dct
+  mark_ensemble: false
+  rank: 2
+  degree: 5
+  L_timepoints: 256
+  n_freqs: 8
+  mark_mlp_dim: 256
+  hidden_act: swish
+  initializer_range: 0.02