Upload folder using huggingface_hub

best_hydra_mark-v1.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43949d03c1052bd9e9a3be70f31db69c4d22040b95bf2c8caf919198755cb964
+size 367591404

best_hydra_mark-v2.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c17b685f26c1e2aa2f71da30f0de86595872a570ffcdb2202b180349d5d6ff0f
+size 238130643

best_hydra_mark.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69dcf929e47fb76a0e94bc901fcb5e0053b91470589923f47278dc397493229f
+size 261410003

lightning_logs/version_0/events.out.tfevents.1775782985.b85934d9bf78.17981.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d0a3291cdf35419efc6c4a90b33b8b444f7b286375678372b7e614a901a8c47
+size 19848

lightning_logs/version_0/hparams.yaml

@@ -0,0 +1,89 @@
+batch_size: 112
+epochs: 3
+stage: 1
+unfrozen_ratio: 0.3
+ckpt_weights_only: false
+checkpoint_dir: ./checkpoints/hydra_mark
+train_data_dir: ./data/train_shards1
+val_data_dir: ./data/val_shards
+weights_path: ./models/hydra_hypernet_mark.pt
+shuffle: true
+use_early_stopping: false
+max_patience_counter: 5
+min_delta: 0.01
+use_gradient_clipping: true
+gradient_clipping_norm: 1.0
+pad_length: 4096
+learning_rate_mark: 0.0006
+learning_rate_hydra: 3.0e-05
+learning_rate_cls: 0.0001
+no_cache: false
+num_workers: 8
+matmul_precision: high
+multi_shot: false
+intervals: 3
+is_prenorm: false
+accumulate_grad_batches: 2
+cart: true
+cart_p: 0.45
+cart_scale: 1.0
+distillation: false
+lr_scheduler:
+  type: cosine
+  warmup_steps: 720
+  total_steps: 14400
+  min_lr_ratio: 0.1
+  polynomial:
+    end_lr_ratio: 0.0
+    power: 1.0
+  plateau:
+    factor: 0.5
+    patience: 3
+    min_lr: 1.0e-06
+trainer:
+  accelerator: gpu
+  devices: -1
+  check_val_every_n_epoch: null
+  num_sanity_val_steps: 0
+  accumulate_grad_batches: 1
+  precision: bf16-true
+  enable_checkpointing: true
+  default_root_dir: ./checkpoints/hydra_mark
+wandb:
+  project: hydra-training_hypernet
+  model_name: HydraForMaskedLM
+  watch_log: all
+  log_freq: 20
+hydra_config:
+  hidden_size: 768
+  vocab_size: 30522
+  type_vocab_size: 2
+  pad_token_id: 0
+  use_position_embeddings: false
+  max_position_embeddings: 4096
+  use_timestep_embeddings: true
+  layer_norm_eps: 1.0e-12
+  dropout: 0.0
+  max_timestep_embeddings: 1000
+  current_timestep: 0
+  d_state: 64
+  d_conv: 7
+  head_dim: 64
+  expand: 2
+  chunk_size: 256
+  is_prenorm: false
+  use_eff_compute: false
+  gradient_checkpointing: true
+  num_hidden_layers: 23
+  guider_hidden_layers: 12
+  device: cpu
+  pool_all: false
+  mark_kernel: hypernet
+  mark_ensemble: false
+  rank: 2
+  degree: 5
+  L_timepoints: 256
+  n_freqs: 8
+  mark_mlp_dim: 256
+  hidden_act: swish
+  initializer_range: 0.02

lightning_logs/version_1/events.out.tfevents.1776275720.b85934d9bf78.42893.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8075de978b2476667852f5875cc2ad8d55f6be6d3806729ac9e442a7db4f039a
+size 20236

lightning_logs/version_1/hparams.yaml

@@ -0,0 +1,89 @@
+batch_size: 112
+epochs: 5
+stage: 2
+unfrozen_ratio: 0.3
+ckpt_weights_only: true
+checkpoint_dir: ./checkpoints/hydra_mark
+train_data_dir: ./data/train_shards2
+val_data_dir: ./data/val_shards
+weights_path: ./models/hydra_hypernet_mark.pt
+shuffle: true
+use_early_stopping: false
+max_patience_counter: 5
+min_delta: 0.01
+use_gradient_clipping: true
+gradient_clipping_norm: 1.0
+pad_length: 4096
+learning_rate_mark: 0.0006
+learning_rate_hydra: 3.0e-05
+learning_rate_cls: 0.0001
+no_cache: false
+num_workers: 8
+matmul_precision: high
+multi_shot: false
+intervals: 3
+is_prenorm: false
+accumulate_grad_batches: 2
+cart: true
+cart_p: 0.45
+cart_scale: 1.0
+distillation: false
+lr_scheduler:
+  type: cosine
+  warmup_steps: 720
+  total_steps: 14400
+  min_lr_ratio: 0.1
+  polynomial:
+    end_lr_ratio: 0.0
+    power: 1.0
+  plateau:
+    factor: 0.5
+    patience: 3
+    min_lr: 1.0e-06
+trainer:
+  accelerator: gpu
+  devices: -1
+  check_val_every_n_epoch: null
+  num_sanity_val_steps: 0
+  accumulate_grad_batches: 1
+  precision: bf16-true
+  enable_checkpointing: true
+  default_root_dir: ./checkpoints/hydra_mark
+wandb:
+  project: hydra-training_hypernet
+  model_name: HydraForMaskedLM
+  watch_log: all
+  log_freq: 20
+hydra_config:
+  hidden_size: 768
+  vocab_size: 30522
+  type_vocab_size: 2
+  pad_token_id: 0
+  use_position_embeddings: false
+  max_position_embeddings: 4096
+  use_timestep_embeddings: true
+  layer_norm_eps: 1.0e-12
+  dropout: 0.0
+  max_timestep_embeddings: 1000
+  current_timestep: 0
+  d_state: 64
+  d_conv: 7
+  head_dim: 64
+  expand: 2
+  chunk_size: 256
+  is_prenorm: false
+  use_eff_compute: false
+  gradient_checkpointing: true
+  num_hidden_layers: 23
+  guider_hidden_layers: 12
+  device: cpu
+  pool_all: false
+  mark_kernel: hypernet
+  mark_ensemble: false
+  rank: 2
+  degree: 5
+  L_timepoints: 256
+  n_freqs: 8
+  mark_mlp_dim: 256
+  hidden_act: swish
+  initializer_range: 0.02

lightning_logs/version_2/events.out.tfevents.1776329596.b85934d9bf78.46840.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0c7420a7b2e8a3090ded89e5d94f25a009c120e2d0f8a28877911c34cf0fc00
+size 7206

lightning_logs/version_2/hparams.yaml

@@ -0,0 +1,89 @@
+batch_size: 112
+epochs: 3
+stage: 3
+unfrozen_ratio: 0.3
+ckpt_weights_only: true
+checkpoint_dir: ./checkpoints/hydra_mark
+train_data_dir: ./data/train_shards3
+val_data_dir: ./data/val_shards
+weights_path: ./models/hydra_hypernet_mark.pt
+shuffle: true
+use_early_stopping: false
+max_patience_counter: 5
+min_delta: 0.01
+use_gradient_clipping: true
+gradient_clipping_norm: 1.0
+pad_length: 4096
+learning_rate_mark: 0.0006
+learning_rate_hydra: 3.0e-05
+learning_rate_cls: 0.0001
+no_cache: false
+num_workers: 8
+matmul_precision: high
+multi_shot: false
+intervals: 3
+is_prenorm: false
+accumulate_grad_batches: 2
+cart: true
+cart_p: 0.45
+cart_scale: 1.0
+distillation: false
+lr_scheduler:
+  type: cosine
+  warmup_steps: 720
+  total_steps: 14400
+  min_lr_ratio: 0.1
+  polynomial:
+    end_lr_ratio: 0.0
+    power: 1.0
+  plateau:
+    factor: 0.5
+    patience: 3
+    min_lr: 1.0e-06
+trainer:
+  accelerator: gpu
+  devices: -1
+  check_val_every_n_epoch: null
+  num_sanity_val_steps: 0
+  accumulate_grad_batches: 1
+  precision: bf16-true
+  enable_checkpointing: true
+  default_root_dir: ./checkpoints/hydra_mark
+wandb:
+  project: hydra-training_hypernet
+  model_name: HydraForMaskedLM
+  watch_log: all
+  log_freq: 20
+hydra_config:
+  hidden_size: 768
+  vocab_size: 30522
+  type_vocab_size: 2
+  pad_token_id: 0
+  use_position_embeddings: false
+  max_position_embeddings: 4096
+  use_timestep_embeddings: true
+  layer_norm_eps: 1.0e-12
+  dropout: 0.0
+  max_timestep_embeddings: 1000
+  current_timestep: 0
+  d_state: 64
+  d_conv: 7
+  head_dim: 64
+  expand: 2
+  chunk_size: 256
+  is_prenorm: false
+  use_eff_compute: false
+  gradient_checkpointing: true
+  num_hidden_layers: 23
+  guider_hidden_layers: 12
+  device: cpu
+  pool_all: false
+  mark_kernel: hypernet
+  mark_ensemble: false
+  rank: 2
+  degree: 5
+  L_timepoints: 256
+  n_freqs: 8
+  mark_mlp_dim: 256
+  hidden_act: swish
+  initializer_range: 0.02