Upload flexitok_superset_albert_w_xglm/config.yaml with huggingface_hub

flexitok_superset_albert_w_xglm/config.yaml

@@ -0,0 +1,228 @@
+name: flexitok_superset_albert_w_xglm
+dump_dir: /fsx/craffel/lingua_logs/flexitok_superset_albert_w_xglm
+seed: 777
+grad_acc_steps: 16
+gc_collect_freq: 1000
+probe_freq: null
+steps: 100000
+data:
+  root_dir: /scratch/craffel/lingua/data/flexitok/
+  sources:
+    fw_edu: 0.4
+    dan_Latn: 0.0216582869670702
+    swe_Latn: 0.0216359765418466
+    vie_Latn: 0.0197485510268674
+    hun_Latn: 0.0247194573562308
+    fas_Arab: 0.0205634624231076
+    tur_Latn: 0.0235455794841729
+    ces_Latn: 0.0248024455266208
+    arb_Arab: 0.0234323706569333
+    ell_Grek: 0.0233670886888026
+    ind_Latn: 0.0269322054593488
+    nld_Latn: 0.0277796326621489
+    pol_Latn: 0.0294120104572311
+    por_Latn: 0.0301413168306825
+    ita_Latn: 0.0324056371021865
+    jpn_Jpan: 0.03553104151369
+    fra_Latn: 0.0381835560678536
+    spa_Latn: 0.0387222793083669
+    deu_Latn: 0.0419925340453022
+    cmn_Hani: 0.0454067521384114
+    rus_Cyrl: 0.0500198157431261
+  batch_size: 2
+  seq_len: 4096
+  n_views: 2
+  seed: 42
+  add_bos: true
+  add_eos: true
+  load_async: true
+  prefetch_size: 1024
+  tokenizer:
+    name: supertokenizer
+    path: meta-llama/Llama-3.2-1B
+    tokenizers:
+    - name: huggingface
+      path: flexitok/reindexed-CohereLabs-aya-expanse-8b
+      load_supermapping: false
+    - name: huggingface
+      path: flexitok/reindexed-bigscience-bloom
+      load_supermapping: false
+    - name: huggingface
+      path: flexitok/reindexed-common-pile-comma-v0.1
+      load_supermapping: false
+    - name: huggingface
+      path: flexitok/reindexed-google-gemma-2-2b
+      load_supermapping: false
+    - name: huggingface
+      path: flexitok/reindexed-gpt2
+      load_supermapping: false
+    - name: huggingface
+      path: flexitok/reindexed-meta-llama-Llama-3.2-1B
+      load_supermapping: false
+    - name: huggingface
+      path: flexitok/reindexed-google-bert-bert-base-multilingual-cased
+      load_supermapping: false
+    - name: huggingface
+      path: flexitok/reindexed-microsoft-Phi-3-mini-4k-instruct
+      load_supermapping: false
+    - name: huggingface
+      path: flexitok/reindexed-Qwen-Qwen3-8B
+      load_supermapping: false
+    - name: huggingface
+      path: facebook/xglm-564M
+      load_supermapping: true
+    - name: huggingface
+      path: google/byt5-small
+      load_supermapping: true
+    - name: tiktoken
+      path: gpt-4o
+      load_supermapping: true
+    - name: tekken
+      path: tekken
+      load_supermapping: true
+    - name: tokenmonster
+      path: englishcode-32000-consistent-v1
+      load_supermapping: true
+    load_supermapping: false
+    dropout: 0.0
+    seed: 42
+optim:
+  lr: 0.001
+  weight_decay: 0.1
+  epsilon: 1.0e-08
+  beta1: 0.9
+  beta2: 0.95
+  clip: 1.0
+  scheduler: cosine
+  warmup: 2000
+  lr_min_ratio: 1.0e-06
+  cycle_length: 1.0
+  cosine_theta: 1.0
+  annealing_step: 1000
+  decay_fraction: 0.1
+  exp_factor: 0.5
+model:
+  dim: 2048
+  n_layers: 25
+  head_dim: null
+  n_heads: 16
+  n_kv_heads: null
+  ffn_dim_multiplier: null
+  multiple_of: 256
+  norm_eps: 1.0e-05
+  rope_theta: 10000.0
+  init_base_std: null
+  init_std_factor: disabled
+  max_seqlen: 4096
+  seed: 42
+  vocab_size: 851586
+  weight_tying: false
+  sliding_window: null
+  use_factorized_embeddings: true
+  factorized_embedding_dim: 512
+distributed:
+  dp_shard: 1
+  dp_replicate: 8
+  tp_size: 1
+  selective_activation_checkpointing: false
+  compile: true
+  fsdp_type: full_shard
+  model_dtype: bf16
+  float8_recipe: null
+  float8_filter: layers\.[0-9]+\.
+  matmul_allow_tf32: false
+  detect_anomaly: false
+  compile_cache_size_limit: 8
+  spawn_method: forkserver
+env:
+  MKL_SERVICE_FORCE_INTEL: GNU
+  OMP_NUM_THREADS: '1'
+  MKL_NUM_THREADS: '1'
+  ENABLE_INTRA_NODE_COMM: '1'
+  TORCH_NCCL_AVOID_RECORD_STREAMS: '1'
+  NCCL_IB_TIMEOUT: '22'
+  NCCL_DEBUG: INFO
+  TORCH_NCCL_ASYNC_ERROR_HANDLING: '1'
+checkpoint:
+  dump:
+    every: 10000
+    keep: -1
+  eval:
+    every: 10000
+    keep: -1
+  path: /fsx/craffel/lingua_logs/flexitok_superset_albert_w_xglm/checkpoints
+  init_ckpt_path: null
+  load_init_optimizer_state: false
+  save_init_ckpt: false
+profiling:
+  run: true
+  trace_folder: profiling
+  mem_warmup: 0
+  mem_steps: 4
+  profile_warmup: 100
+  profile_steps: 4
+logging:
+  freq: 1
+  acc_freq: null
+  wandb: null
+async_eval_gpus: 8
+eval:
+  harness:
+    tasks:
+    - hellaswag
+    - piqa
+    - arc_easy
+    - arc_challenge
+    - include_base_44_arabic
+    - include_base_44_chinese
+    - include_base_44_german
+    - include_base_44_greek
+    - include_base_44_persian
+    - include_base_44_french
+    - include_base_44_hungarian
+    - include_base_44_indonesian
+    - include_base_44_italian
+    - include_base_44_japanese
+    - include_base_44_dutch
+    - include_base_44_polish
+    - include_base_44_portuguese
+    - include_base_44_russian
+    - include_base_44_spanish
+    - include_base_44_turkish
+    - include_base_44_vietnamese
+    - belebele_arb_Arab
+    - belebele_ces_Latn
+    - belebele_zho_Hans
+    - belebele_dan_Latn
+    - belebele_deu_Latn
+    - belebele_ell_Grek
+    - belebele_pes_Arab
+    - belebele_fra_Latn
+    - belebele_hun_Latn
+    - belebele_ind_Latn
+    - belebele_ita_Latn
+    - belebele_jpn_Jpan
+    - belebele_nld_Latn
+    - belebele_pol_Latn
+    - belebele_por_Latn
+    - belebele_rus_Cyrl
+    - belebele_spa_Latn
+    - belebele_swe_Latn
+    - belebele_tur_Latn
+    - belebele_vie_Latn
+    - belebele_eng_Latn
+    - xnli_ar
+    - xnli_zh
+    - xnli_de
+    - xnli_el
+    - xnli_en
+    - xnli_es
+    - xnli_fr
+    - xnli_hi
+    - xnli_ru
+    - xnli_tr
+    - xnli_vi
+  generator:
+    max_tokens: 16384
+    dtype: bf16
+    add_bos: false