Upload hydra_cfg.yaml with huggingface_hub

hydra_cfg.yaml

@@ -0,0 +1,343 @@
+RC_augmentation: false
+_dataset_cfg_lookup:
+  dlb_cmp_gm12878:
+    eval_split: validation
+    hf_path: jzshared/dlb_cmp_gm12878
+    label_key: label_ut
+    mask_key: mask_ut
+    num_workers: 0
+    path: data/dlb_cmp_gm12878
+    pin_memory: true
+    reference_id: hg38
+    sequence_format: string
+    sequence_key: sequence
+    shuffle: true
+    test_split: test
+    train_split: train
+    type: cmp_seq
+  dlb_cmp_h1hesc:
+    eval_split: validation
+    hf_path: jzshared/dlb_cmp_h1hesc
+    label_key: label_ut
+    mask_key: mask_ut
+    num_workers: 0
+    path: data/dlb_cmp_h1hesc
+    pin_memory: true
+    reference_id: hg38
+    sequence_format: string
+    sequence_key: sequence
+    shuffle: true
+    test_split: test
+    train_split: train
+    type: cmp_seq
+  dlb_cmp_hct116:
+    eval_split: validation
+    hf_path: jzshared/dlb_cmp_hct116
+    label_key: label_ut
+    mask_key: mask_ut
+    num_workers: 0
+    path: data/dlb_cmp_hct116
+    pin_memory: true
+    reference_id: hg38
+    sequence_format: string
+    sequence_key: sequence
+    shuffle: true
+    test_split: test
+    train_split: train
+    type: cmp_seq
+  dlb_cmp_hff:
+    eval_split: validation
+    hf_path: jzshared/dlb_cmp_hff
+    label_key: label_ut
+    mask_key: mask_ut
+    num_workers: 0
+    path: data/dlb_cmp_hff
+    pin_memory: true
+    reference_id: hg38
+    sequence_format: string
+    sequence_key: sequence
+    shuffle: true
+    test_split: test
+    train_split: train
+    type: cmp_seq
+  dlb_cmp_imr90:
+    eval_split: validation
+    hf_path: jzshared/dlb_cmp_imr90
+    label_key: label_ut
+    mask_key: mask_ut
+    num_workers: 0
+    path: data/dlb_cmp_imr90
+    pin_memory: true
+    reference_id: hg38
+    sequence_format: string
+    sequence_key: sequence
+    shuffle: true
+    test_split: test
+    train_split: train
+    type: cmp_seq
+  gencode128k_basic:
+    hf_path: jzshared/gencode128k_basic
+    path: data/gencode128k_basic
+    type: refseq
+  gencode128k_debug:
+    hf_path: jzshared/gencode128k_debug
+    path: data/gencode128k_debug
+    type: refseq
+  gencode_human_12.8k:
+    hf_path: jzshared/gencode_human_12.8k
+    path: data/gencode_human_12.8k
+    type: refseq
+  gencode_human_128k:
+    hf_path: jzshared/gencode_human_128k
+    path: data/gencode_human_128k
+    type: refseq
+  hg38_128k:
+    hf_path: jzshared/hg38_cds_anchored_128000
+    path: data/hg38_cds_anchored_128000
+    type: refseq
+  hg38_12k:
+    hf_path: jzshared/hg38_12800
+    path: data/hg38_cds_anchored_len12800_mincds150_1000000samples
+    type: refseq
+  hg38_cds_4m:
+    hf_path: null
+    path: data/hg38_cds_dataset_4m_filtered
+    type: refseq
+  orca32m_cmp_seq:
+    eval_split: validation
+    hf_path: jzshared/orca32m_cmp
+    label_key: label_ut
+    mask_key: mask_ut
+    num_workers: 0
+    path: data/orca32m_cmp_seq
+    pin_memory: true
+    reference_id: hg38
+    sequence_format: string
+    sequence_key: sequence
+    shuffle: true
+    test_split: test
+    train_split: train
+    type: cmp_seq
+_unimportant_cfg:
+  fields:
+  - gpus
+  - debug
+  - wandb
+  - env
+  - uid
+  - local_rank
+  - is_distributed
+  - master_port
+  - device_type
+  - cluster
+  - world_size
+  - train_dataset
+  - eval_datasets
+  - user_cfg
+  - rank
+  - device
+  - hf_access_token
+  - hf_private
+  - hf_repo
+  - hf_user
+  - hf_token
+  - save_every
+  - eval_steps
+  - save_steps
+  - upload_to_hf
+  - logging
+  - log_every
+  - use_wandb
+  - project_root
+  - version
+  postfix:
+  - _path
+  - _file
+  - _dir
+  - _alias
+  - _prefix
+  prefix:
+  - _
+alias: GeneZip-CMP-7R-LT-32BPT-100B
+alpha_exp: 1.0
+alpha_max: 0.03
+arch: hnet
+batch_size: 32
+bp_per_token: 32
+cluster: mila
+cmd: python src/scripts/train_genezip_v1.py exp=glm/stage1 data=gencode_human_12.8k
+  model=hnet/mamba_64m_2dc use_routing_floor=true use_routing_ceiling=true r_hi=0.3
+  r_low=0.3 max_routing_tokens=[5000,2000] max_len=12800 batch_size=32 grad_acc_steps=1
+  max_train_steps=30600 eval_steps=500 num_valid_samples=3000 upload_to_hf=true wandb.project=DNAFM_v2
+  bp_per_token=32 region_info=promoter8_cds16_utr16_exon16_intron4_nig2_dig4 alias=GeneZip-CMP-7R-LT-32BPT-100B
+  use_wandb=true hf_repo=jzshared/GeneZip-CMP-7R-LT-32BPT-100B
+config_path: null
+data: gencode_human_12.8k
+data_alias: ${.data}_${max_len}
+dataset: ${_dataset_cfg_lookup[${data}]}
+device: cuda
+device_type: GPU
+dirs:
+  data_cache: ${project_root}/data_cache/
+  data_storage: ${project_root}/data/
+  hydra: ${project_root}/temp/hydra/
+  output: ${project_root}/output/${data_alias}/${alias}/
+  temp: ${project_root}/temp/working_dir/${uid}/
+  wandb_cache: ${oc.env:WANDB_CACHE_DIR,${project_root}/temp/wandb_cache/}
+epochs: 200
+eval_batch_size: ${batch_size}
+eval_steps: 500
+grad_acc_steps: 1
+hf_private: false
+hf_repo: jzshared/GeneZip-CMP-7R-LT-32BPT-100B
+hf_user: jzshared
+is_distributed: true
+local_rank: 0
+log_every: 10
+logging:
+  level: info
+  log_wandb_metric_to_stdout: true
+lr: 0.001
+master_port: '49381'
+max_data_samples: null
+max_eval_samples: ${num_valid_samples}
+max_grad_norm: 2.0
+max_len: 12800
+max_length: ${max_len}
+max_routing_tokens:
+- 5000
+- 2000
+max_train_steps: 30600
+min_routing_tokens: 8
+mixed_precision: bf16
+mode: Formal
+model:
+  arch: hnet
+  name: hnet_mamba_64m_2dc
+model_alias: ${oc.select:model.name,UnknownModel}
+model_cfg:
+  arch_layout:
+  - m2
+  - - m2
+    - - m15
+    - m2
+  - m2
+  attn_cfg:
+    num_heads:
+    - 8
+    - 8
+    - 12
+    rotary_emb_dim:
+    - 16
+    - 16
+    - 24
+    window_size:
+    - 511
+    - 511
+    - -1
+  d_intermediate:
+  - 0
+  - 0
+  - 2048
+  d_model:
+  - 512
+  - 512
+  - 768
+  max_routing_tokens: ${max_routing_tokens}
+  min_routing_tokens: ${min_routing_tokens}
+  n_gpt: 1.0
+  r_hi: ${r_hi}
+  r_low: ${r_low}
+  r_warm_up_end: ${r_warm_up_end}
+  r_warm_up_start: ${r_warm_up_start}
+  ssm_cfg:
+    chunk_size: 256
+    d_conv: 4
+    d_state: 64
+    expand: 2
+    head_dim: 64
+  tie_embeddings: true
+  vocab_size: 12
+name: hnet_base
+num_test_samples: 0
+num_train_samples: 0
+num_valid_samples: 3000
+pretrained_ckpt: null
+project_root: ${hydra:runtime.cwd}
+r_hi: 0.3
+r_low: 0.3
+r_warm_up_end: 750
+r_warm_up_start: 200
+rank: 0
+reference_loss: null
+region_info: promoter8_cds16_utr16_exon16_intron4_nig2_dig4
+region_label_map:
+  cds: 1
+  dig: 6
+  exon: 3
+  intron: 4
+  nig: 5
+  promoter: 0
+  utr: 2
+save_steps: 3000
+seed: 0
+source: ${dataset.type}
+stop_steps: 0
+strictness_exp: 1.0
+strictness_max: 1.0
+tokenizer: fast
+train_steps: 9999999
+training:
+  adam_beta1: 0.9
+  adam_beta2: 0.95
+  bf16: true
+  dataloader_drop_last: true
+  dataloader_num_workers: 1
+  disable_tqdm: false
+  do_train: true
+  eval_steps: ${eval_steps}
+  eval_strategy: steps
+  gradient_accumulation_steps: ${grad_acc_steps}
+  gradient_checkpointing: false
+  group_by_length: false
+  hnet_initializer_range: 0.02
+  hnet_lr_multiplier: null
+  label_names:
+  - input_ids
+  learning_rate: ${lr}
+  logging_steps: ${log_every}
+  lr_scheduler_type: linear
+  max_grad_norm: ${max_grad_norm}
+  max_train_steps: ${max_train_steps}
+  num_train_epochs: ${epochs}
+  output_dir: ${dirs.output}
+  overrides: {}
+  per_device_eval_batch_size: ${eval_batch_size}
+  per_device_train_batch_size: ${batch_size}
+  remove_unused_columns: false
+  report_to: null
+  save_steps: ${save_steps}
+  save_strategy: steps
+  use_lr_multiplier: true
+  warmup_steps: 500
+  weight_decay: 0.1
+training_alias: ${mode}_glm_s1_${region_info}_bp${bp_per_token}_aw${warmup_steps}_amax${alpha_max}_smax${strictness_max}_lr${lr}_e${epochs}_ms${max_train_steps}_maxlen${max_len}
+uid: p2nb6whc
+upload_to_hf: true
+use_routing_ceiling: true
+use_routing_floor: true
+use_wandb: true
+valid_test_downsample: null
+version: NA
+wandb:
+  dir: ${dirs.wandb_cache}
+  entity: ${oc.select:env.vars.wandb_entity,${oc.env:WANDB_ENTITY,null}}
+  id: p2nb6whc
+  mode: online
+  name: GeneZip-CMP-7R-LT-32BPT-100B
+  project: DNAFM_v2
+  step_metric: null
+  tags: []
+  url: https://wandb.ai/jzshared/DNAFM_v2/runs/p2nb6whc
+warmup_steps: 0
+world_size: 8