Create hydra_cfg.yaml

hydra_cfg.yaml

@@ -0,0 +1,243 @@
+# @package _global_
+seed: 0
+mode: Stage1
+alias: GeneZip-7R_12.8K-100B
+data_alias: ${.data}_${max_len}
+model_alias: ${oc.select:model.name,UnknownModel}
+uid: ofagcfej
+project_root: ${hydra:runtime.cwd}
+version: NA
+device: cuda
+use_routing_floor: true
+cluster: mila
+dirs:
+  data_cache: ${project_root}/data_cache/
+  data_storage: ${project_root}/data/
+  temp: ${project_root}/temp/working_dir/${uid}/
+  output: ${project_root}/output/${data_alias}/${alias}/
+  hydra: ${project_root}/temp/hydra/
+  wandb_cache: ${oc.env:WANDB_CACHE_DIR,${project_root}/temp/wandb_cache/}
+use_wandb: true
+wandb:
+  name: GeneZip-7R_12.8K-100B
+  project: ${oc.select:env.vars.wandb_proj,DNAFM}
+  entity: ${oc.select:env.vars.wandb_entity,${oc.env:WANDB_ENTITY,null}}
+  tags:
+  - ${mode}
+  dir: ${dirs.wandb_cache}
+  mode: online
+  id: ofagcfej
+  step_metric: null
+  url: https://wandb.ai/liuxxiiiii/DNAFM/runs/ofagcfej
+logging:
+  level: info
+  log_wandb_metric_to_stdout: true
+max_data_samples: null
+data: gencode_human_12.8k
+dataset: ${_dataset_cfg_lookup[${data}]}
+max_len: 12800
+source: ${dataset.type}
+max_length: ${max_len}
+valid_test_downsample: null
+tokenizer: fast
+RC_augmentation: false
+reference_loss: null
+hf_token: ${oc.env:HUGGINGFACE_HUB_TOKEN,null}
+_dataset_cfg_lookup:
+  hg38_cds_4m:
+    type: refseq
+    path: data/hg38_cds_dataset_4m_filtered
+    hf_path: null
+  hg38_12k:
+    type: refseq
+    path: data/hg38_cds_anchored_len12800_mincds150_1000000samples
+    hf_path: jzshared/hg38_12800
+  hg38_128k:
+    type: refseq
+    path: data/hg38_cds_anchored_128000
+    hf_path: jzshared/hg38_cds_anchored_128000
+  gencode128k_debug:
+    type: refseq
+    path: data/gencode128k_debug
+    hf_path: jzshared/gencode128k_debug
+  gencode_human_12.8k:
+    type: refseq
+    path: data/gencode_human_12.8k
+    hf_path: jzshared/gencode_human_12.8k
+  gencode_human_128k:
+    type: refseq
+    path: data/gencode_human_128k
+    hf_path: jzshared/gencode_human_128k
+  gencode128k_basic:
+    type: refseq
+    path: data/gencode128k_basic
+    hf_path: jzshared/gencode128k_basic
+hf_repo: jzshared/GeneZip-7R_12.8K-100B
+hf_repo_owner: jzshared
+upload_to_hf: true
+private: false
+name: hnet_base
+training_alias: ${mode}_glm_s1_${region_info}_bp${bp_per_token}_aw${warmup_steps}_amax${alpha_max}_smax${strictness_max}_lr${lr}_e${epochs}_ms${max_train_steps}_maxlen${max_len}
+lr: 4.068919838510263e-08
+grad_acc_steps: 1
+eval_steps: 500
+save_steps: 1000
+epochs: 200
+max_train_steps: 30600
+max_eval_samples: 1000
+batch_size: 32
+eval_batch_size: ${batch_size}
+training:
+  learning_rate: ${lr}
+  do_train: true
+  group_by_length: false
+  remove_unused_columns: false
+  label_names:
+  - input_ids
+  disable_tqdm: false
+  num_train_epochs: ${epochs}
+  max_train_steps: ${max_train_steps}
+  eval_strategy: steps
+  eval_steps: ${eval_steps}
+  save_strategy: steps
+  save_steps: ${save_steps}
+  logging_steps: 10
+  report_to: null
+  per_device_train_batch_size: ${batch_size}
+  per_device_eval_batch_size: ${eval_batch_size}
+  gradient_accumulation_steps: ${grad_acc_steps}
+  dataloader_num_workers: 1
+  dataloader_drop_last: true
+  gradient_checkpointing: false
+  max_grad_norm: 2.0
+  weight_decay: 0.1
+  use_lr_multiplier: true
+  lr_scheduler_type: linear
+  warmup_steps: 500
+  adam_beta1: 0.9
+  adam_beta2: 0.95
+  bf16: true
+  overrides: {}
+  output_dir: ${dirs.output}
+region_info: promoter1_cds2_utr4_exon4_intron16_nig8_dig32
+bp_per_token: 128
+warmup_steps: 0
+alpha_max: 0.03
+alpha_exp: 1.0
+strictness_max: 1.0
+strictness_exp: 1.0
+arch: hnet
+config_path: null
+model:
+  name: hnet_mamba_64m_2dc
+  arch: hnet
+r_hi: 0.3
+r_low: 0.0
+min_routing_tokens: 8
+r_warm_up_start: 200
+r_warm_up_end: 750
+model_cfg:
+  arch_layout:
+  - m2
+  - - m2
+    - - m15
+    - m2
+  - m2
+  n_gpt: 1.0
+  d_model:
+  - 512
+  - 512
+  - 768
+  d_intermediate:
+  - 0
+  - 0
+  - 2048
+  vocab_size: 12
+  ssm_cfg:
+    chunk_size: 256
+    head_dim: 64
+    d_conv: 4
+    d_state: 64
+    expand: 2
+  attn_cfg:
+    num_heads:
+    - 8
+    - 8
+    - 12
+    rotary_emb_dim:
+    - 16
+    - 16
+    - 24
+    window_size:
+    - 511
+    - 511
+    - -1
+  tie_embeddings: true
+  r_hi: ${r_hi}
+  r_low: ${r_low}
+  min_routing_tokens: ${min_routing_tokens}
+  r_warm_up_start: ${r_warm_up_start}
+  r_warm_up_end: ${r_warm_up_end}
+rank: 0
+local_rank: 0
+world_size: 8
+is_distributed: true
+device_type: GPU
+cmd: python src/scripts/train_genezip_v1.py exp=glm/stage1 data=gencode_human_12.8k
+  model=hnet/mamba_64m_2dc max_len=12800 batch_size=32 grad_acc_steps=1 max_train_steps=30600
+  eval_steps=500 bp_per_token=128 region_info=promoter1_cds2_utr4_exon4_intron16_nig8_dig32
+  alias=GeneZip-7R_12.8K-100B use_wandb=true upload_to_hf=true hf_repo=jzshared/GeneZip-7R_12.8K-100B
+master_port: '40617'
+summary:
+  model_size: 69.8M
+  max_gpu_mem: 55.857
+model_size: 69.8M
+max_gpu_mem: 55.857
+loss: 1.0907
+grad_norm: 0.03689846768975258
+loss_ce: 1.0496642589569092
+router_selected_tokens_s0: 460.53125
+router_trigger_rate_s0: 0.0
+router_avg_deficit_s0: 0.0
+router_selected_tokens_s1: 17.875
+router_trigger_rate_s1: 0.03125
+router_avg_deficit_s1: 1.0
+loss_region: 0.060342203825712204
+comp_strictness: 1.0
+comp_rl_weight: 0.03
+loss_total: 1.1100064516067505
+tokens_trained: 100.260990464
+epoch: 8.680851063829786
+step: 30600
+eval_runtime: 0.5367
+eval_ppl: 2.8175418770540306
+eval_bp_per_token_promoter: 261.42059336823735
+eval_bp_per_token_cds: 332.3425925925926
+eval_bp_per_token_utr: 752.6538461538462
+eval_bp_per_token_exon: 690.0
+eval_bp_per_token_intron: 1128.3223388305846
+eval_bp_per_token_nig: 1050.814049586777
+eval_ppl_promoter: 3.124982450778065
+eval_ppl_cds: 3.7295588949788616
+eval_ppl_utr: 3.440777494381466
+eval_ppl_exon: 3.106529824327725
+eval_ppl_intron: 2.9497514633424493
+eval_ppl_nig: 2.570548504715622
+eval_F: 0.0013347396932051983
+eval_G: 0.004359940369588607
+eval_avg_bp_per_token: 749.209756097561
+eval_F_promoter: 0.003825253347931159
+eval_G_promoter: 0.009358177417820473
+eval_F_cds: 0.0030089432479870728
+eval_G_cds: 0.007192112232886635
+eval_F_utr: 0.0013286320200316829
+eval_G_utr: 0.004329032845316572
+eval_F_exon: 0.0014492753623188406
+eval_G_exon: 0.004401843674516908
+eval_F_intron: 0.0008862715605156054
+eval_G_intron: 0.0035734517902154027
+eval_F_nig: 0.0009516431574104296
+eval_G_nig: 0.003343231843572673
+train_runtime: 10669.8105
+total_flos: 0.0
+train_loss: 1.2369615290523355