Upload config.yaml with huggingface_hub

config.yaml

@@ -0,0 +1,70 @@
+# ELECTRA Corrective MLM — full-scale 1B pretraining
+# Generator corrupts input with plausible-but-guaranteed-wrong tokens,
+# discriminator predicts original tokens at ALL positions.
+#
+# Usage:
+#   /cache/ssahn_lab/envs/voxfaplm/bin/python -m torch.distributed.run \
+#       --nproc_per_node=8 experiments/train_multinode.py \
+#       --config configs/pretrain/pretrain_electra.yaml
+
+model:
+  name: "ELECTRAProteinModel"
+  max_residues: 512
+  embed_dim: 480
+  encoder_depth: 10
+  encoder_heads: 20
+  decoder_dim: 480
+  decoder_depth: 2
+  decoder_heads: 20
+  # ELECTRA-specific
+  generator_depth: 8            # ~27% of disc encoder
+  generator_decoder_depth: 2
+  disc_loss_weight: 50.0         # corrective CE ≈ same scale as generator CE
+  generator_lr_multiplier: 1.0
+  generator_temperature: 2.0
+
+# Training configuration
+training:
+  learning_rate: 1.0e-3
+  weight_decay: 0.01
+  warmup_steps: 5000
+  batch_size: 128
+  num_workers: 8
+  max_steps: 500000
+  total_training_steps: 500000
+  accumulate_grad_batches: 1
+  val_check_interval: 100
+  test_interval: 100
+  gradient_clip_val: 1.0
+# Data configuration
+data:
+  name: "ELECTRADataModule"
+  lmdb_dir: "/cache/ssahn_lab/lmdb/atlas_pdb_all_CHI"
+  pdb_dir: "/cache"
+  struct_format: "fullatom"
+  max_length: 512
+  bb_vocab_size: 512
+  fa_vocab_size: 512
+  mask_prob: 0.6
+  random_cropping: true
+  # Sequence packing: concat variable-length samples into one super-sequence
+  # (bsz=1) with xformers BlockDiagonalMask. Keeps flash attention and
+  # eliminates padding waste. Only affects training — probing/eval uses
+  # padded path unchanged.
+  pack_sequences: true
+  # dynamic_batch_tokens: 65536  # opt-in: variable-size batches that pack up
+  #                                to this token budget (maximizes samples per
+  #                                batch based on actual lengths). Disables compile.
+
+# Lightning configuration
+lightning:
+  gpus: 4
+  nodes: 1
+  precision: "bf16-mixed"
+  compile: false
+  checkpoint_dir: "/cache/ssahn_lab/pretrain_model/electra_35M/checkpoints"
+  log_dir: "/cache/ssahn_lab/pretrain_model/electra_35M/logs"
+  wandb_dir: "/cache/ssahn_lab/pretrain_model/electra_35M/wandb"
+  wandb_project: "PLM_submission"  
+  wandb_name: "electra_35M"
+  save_every_n_steps: 10000