checkpoints/safe_v2_2nodes/0000005000/params.json

{"name": "safe_v2_2nodes", "dump_dir": "/fsx/craffel/lingua_logs/safe_v2_2nodes", "seed": 777, "grad_acc_steps": 4, "gc_collect_freq": 1000, "probe_freq": null, "steps": 100000, "data": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "batch_size": 4, "seq_len": 4096, "n_views": 2, "seed": 42, "add_bos": true, "add_eos": true, "load_async": true, "prefetch_size": 1024, "tokenizer": {"name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_arb_Arab_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ces_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000_v2", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_dan_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_deu_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ell_Grek_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_fas_Arab_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_fra_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_hun_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ind_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ita_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_nld_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_pol_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_por_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_swe_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_vie_Latn_8000", "load_supermapping": false}], "load_supermapping": false, "dropout": 0.0, "seed": 42, "superset_code_name": "safe_v2", "n_words": 122554}, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/reindexed-flexitok-bpe_arb_Arab_8000", "ces_Latn": "flexitok/reindexed-flexitok-bpe_ces_Latn_8000", "cmn_Hani": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000", "dan_Latn": "flexitok/reindexed-flexitok-bpe_dan_Latn_8000", "deu_Latn": "flexitok/reindexed-flexitok-bpe_deu_Latn_8000", "ell_Grek": "flexitok/reindexed-flexitok-bpe_ell_Grek_8000", "fas_Arab": "flexitok/reindexed-flexitok-bpe_fas_Arab_8000", "fra_Latn": "flexitok/reindexed-flexitok-bpe_fra_Latn_8000", "fw_edu": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "hun_Latn": "flexitok/reindexed-flexitok-bpe_hun_Latn_8000", "ind_Latn": "flexitok/reindexed-flexitok-bpe_ind_Latn_8000", "ita_Latn": "flexitok/reindexed-flexitok-bpe_ita_Latn_8000", "jpn_Jpan": "flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2", "nld_Latn": "flexitok/reindexed-flexitok-bpe_nld_Latn_8000", "pol_Latn": "flexitok/reindexed-flexitok-bpe_pol_Latn_8000", "por_Latn": "flexitok/reindexed-flexitok-bpe_por_Latn_8000", "rus_Cyrl": "flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000", "spa_Latn": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000", "swe_Latn": "flexitok/reindexed-flexitok-bpe_swe_Latn_8000", "tur_Latn": "flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2", "vie_Latn": "flexitok/reindexed-flexitok-bpe_vie_Latn_8000"}, "task_to_tokenizer": {}, "suitable_tokenizer_probability": 1.0}}, "optim": {"lr": 0.001, "weight_decay": 0.1, "epsilon": 1e-08, "beta1": 0.9, "beta2": 0.95, "clip": 1.0, "scheduler": "cosine", "warmup": 2000, "lr_min_ratio": 1e-06, "cycle_length": 1.0, "cosine_theta": 1.0, "annealing_step": 1000, "decay_fraction": 0.1, "exp_factor": 0.5}, "model": {"dim": 2048, "n_layers": 25, "head_dim": null, "n_heads": 16, "n_kv_heads": null, "ffn_dim_multiplier": null, "multiple_of": 256, "norm_eps": 1e-05, "rope_theta": 10000.0, "init_base_std": null, "init_std_factor": "disabled", "max_seqlen": 4096, "seed": 42, "vocab_size": 122554, "weight_tying": false, "sliding_window": null, "use_factorized_embeddings": false, "factorized_embedding_dim": 0}, "distributed": {"dp_shard": 1, "dp_replicate": 16, "tp_size": 1, "selective_activation_checkpointing": false, "compile": true, "fsdp_type": "full_shard", "model_dtype": "bf16", "float8_recipe": null, "float8_filter": "layers\\.[0-9]+\\.", "matmul_allow_tf32": false, "detect_anomaly": false, "compile_cache_size_limit": 8, "spawn_method": "forkserver"}, "env": {"MKL_SERVICE_FORCE_INTEL": "GNU", "OMP_NUM_THREADS": "1", "MKL_NUM_THREADS": "1", "ENABLE_INTRA_NODE_COMM": "1", "TORCH_NCCL_AVOID_RECORD_STREAMS": "1", "NCCL_IB_TIMEOUT": "22", "NCCL_DEBUG": "INFO", "TORCH_NCCL_ASYNC_ERROR_HANDLING": "1"}, "checkpoint": {"dump": {"every": 2500, "keep": -1}, "eval": {"every": 2500, "keep": -1}, "path": "/fsx/craffel/lingua_logs/checkpoints/safe_v2_2nodes", "init_ckpt_path": null, "load_init_optimizer_state": false, "save_init_ckpt": false}, "profiling": {"run": true, "trace_folder": "profiling", "mem_warmup": 0, "mem_steps": 4, "profile_warmup": 100, "profile_steps": 4}, "logging": {"freq": 1, "acc_freq": null, "wandb": {"job_type": null, "dir": null, "project": "MoTo-Training", "entity": "raffel-reports", "tags": null, "group": null, "name": "safe_v2_2nodes", "notes": null, "config_exclude_keys": null, "config_include_keys": null, "anonymous": null, "mode": null, "allow_val_change": null, "resume": null, "force": null, "tensorboard": null, "sync_tensorboard": null, "monitor_gym": null, "save_code": null, "id": null, "fork_from": null, "resume_from": null}}, "async_eval_gpus": null, "eval": {"harness": {"tasks": ["hellaswag", "piqa", "arc_easy", "arc_challenge", "xnli_ar", "xnli_zh", "xnli_de", "xnli_el", "xnli_en", "xnli_es", "xnli_fr", "xnli_ru", "xnli_tr", "xnli_vi", "toksuite_english", "toksuite_chinese", "toksuite_turkish", "toksuite_italian", "toksuite_farsi", "toksuite_math", "toksuite_stem", "xcopa_id", "xcopa_it", "xcopa_tr", "xcopa_vi", "xcopa_zh", "xstorycloze_ar", "xstorycloze_en", "xstorycloze_es", "xstorycloze_id", "xstorycloze_ru", "xstorycloze_zh", "paws_de", "paws_en", "paws_es", "paws_fr", "paws_ja", "paws_zh", "hellaswag_ar", "hellaswag_da", "hellaswag_de", "hellaswag_fr", "hellaswag_hu", "hellaswag_id", "hellaswag_it", "hellaswag_nl", "hellaswag_pt", "hellaswag_ru", "hellaswag_sv", "hellaswag_vi", "hellaswag_es"]}, "generator": {"max_tokens": 16384, "dtype": "bf16", "add_bos": false}, "routing": {"suitable_tokenizer_probability": 1.0, "task_to_tokenizer": {"arc_easy": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "arc_challenge": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "hellaswag": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "piqa": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "include_base_44_arabic": "flexitok/reindexed-flexitok-bpe_arb_Arab_8000", "include_base_44_chinese": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000", "include_base_44_german": "flexitok/reindexed-flexitok-bpe_deu_Latn_8000", "include_base_44_greek": "flexitok/reindexed-flexitok-bpe_ell_Grek_8000", "include_base_44_persian": "flexitok/reindexed-flexitok-bpe_fas_Arab_8000", "include_base_44_french": "flexitok/reindexed-flexitok-bpe_fra_Latn_8000", "include_base_44_hungarian": "flexitok/reindexed-flexitok-bpe_hun_Latn_8000", "include_base_44_indonesian": "flexitok/reindexed-flexitok-bpe_ind_Latn_8000", "include_base_44_italian": "flexitok/reindexed-flexitok-bpe_ita_Latn_8000", "include_base_44_japanese": "flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2", "include_base_44_dutch": "flexitok/reindexed-flexitok-bpe_nld_Latn_8000", "include_base_44_polish": "flexitok/reindexed-flexitok-bpe_pol_Latn_8000", "include_base_44_portuguese": "flexitok/reindexed-flexitok-bpe_por_Latn_8000", "include_base_44_russian": "flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000", "include_base_44_spanish": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000", "include_base_44_turkish": "flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2", "include_base_44_vietnamese": "flexitok/reindexed-flexitok-bpe_vie_Latn_8000", "belebele_arb_Arab": "flexitok/reindexed-flexitok-bpe_arb_Arab_8000", "belebele_ces_Latn": "flexitok/reindexed-flexitok-bpe_ces_Latn_8000", "belebele_zho_Hans": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000", "belebele_dan_Latn": "flexitok/reindexed-flexitok-bpe_dan_Latn_8000", "belebele_deu_Latn": "flexitok/reindexed-flexitok-bpe_deu_Latn_8000", "belebele_ell_Grek": "flexitok/reindexed-flexitok-bpe_ell_Grek_8000", "belebele_pes_Arab": "flexitok/reindexed-flexitok-bpe_fas_Arab_8000", "belebele_fra_Latn": "flexitok/reindexed-flexitok-bpe_fra_Latn_8000", "belebele_hun_Latn": "flexitok/reindexed-flexitok-bpe_hun_Latn_8000", "belebele_ind_Latn": "flexitok/reindexed-flexitok-bpe_ind_Latn_8000", "belebele_ita_Latn": "flexitok/reindexed-flexitok-bpe_ita_Latn_8000", "belebele_jpn_Jpan": "flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2", "belebele_nld_Latn": "flexitok/reindexed-flexitok-bpe_nld_Latn_8000", "belebele_pol_Latn": "flexitok/reindexed-flexitok-bpe_pol_Latn_8000", "belebele_por_Latn": "flexitok/reindexed-flexitok-bpe_por_Latn_8000", "belebele_rus_Cyrl": "flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000", "belebele_spa_Latn": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000", "belebele_swe_Latn": "flexitok/reindexed-flexitok-bpe_swe_Latn_8000", "belebele_tur_Latn": "flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2", "belebele_vie_Latn": "flexitok/reindexed-flexitok-bpe_vie_Latn_8000", "belebele_eng_Latn": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "xnli_ar": "flexitok/reindexed-flexitok-bpe_arb_Arab_8000", "xnli_zh": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000", "xnli_de": "flexitok/reindexed-flexitok-bpe_deu_Latn_8000", "xnli_el": "flexitok/reindexed-flexitok-bpe_ell_Grek_8000", "xnli_en": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "xnli_es": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000", "xnli_fr": "flexitok/reindexed-flexitok-bpe_fra_Latn_8000", "xnli_ru": "flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000", "xnli_tr": "flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2", "xnli_vi": "flexitok/reindexed-flexitok-bpe_vie_Latn_8000", "regex:toksuite_turkish*": "flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2", "regex:toksuite_english*": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "regex:toksuite_farsi*": "flexitok/reindexed-flexitok-bpe_fas_Arab_8000", "regex:toksuite_italian*": "flexitok/reindexed-flexitok-bpe_ita_Latn_8000", "regex:toksuite_chinese*": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000", "regex:toksuite_math*": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "regex:toksuite_stem*": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "flores200_arb_Arab": "flexitok/reindexed-flexitok-bpe_arb_Arab_8000", "flores200_ces_Latn": "flexitok/reindexed-flexitok-bpe_ces_Latn_8000", "flores200_cmn_Hani": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000", "flores200_dan_Latn": "flexitok/reindexed-flexitok-bpe_dan_Latn_8000", "flores200_deu_Latn": "flexitok/reindexed-flexitok-bpe_deu_Latn_8000", "flores200_ell_Grek": "flexitok/reindexed-flexitok-bpe_ell_Grek_8000", "flores200_fas_Arab": "flexitok/reindexed-flexitok-bpe_fas_Arab_8000", "flores200_fra_Latn": "flexitok/reindexed-flexitok-bpe_fra_Latn_8000", "flores200_fw_edu": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "flores200_eng_Latn": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "flores200_hun_Latn": "flexitok/reindexed-flexitok-bpe_hun_Latn_8000", "flores200_ind_Latn": "flexitok/reindexed-flexitok-bpe_ind_Latn_8000", "flores200_ita_Latn": "flexitok/reindexed-flexitok-bpe_ita_Latn_8000", "flores200_jpn_Jpan": "flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2", "flores200_nld_Latn": "flexitok/reindexed-flexitok-bpe_nld_Latn_8000", "flores200_pol_Latn": "flexitok/reindexed-flexitok-bpe_pol_Latn_8000", "flores200_por_Latn": "flexitok/reindexed-flexitok-bpe_por_Latn_8000", "flores200_rus_Cyrl": "flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000", "flores200_spa_Latn": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000", "flores200_swe_Latn": "flexitok/reindexed-flexitok-bpe_swe_Latn_8000", "flores200_tur_Latn": "flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2", "flores200_vie_Latn": "flexitok/reindexed-flexitok-bpe_vie_Latn_8000", "xcopa_id": "flexitok/reindexed-flexitok-bpe_ind_Latn_8000", "xcopa_it": "flexitok/reindexed-flexitok-bpe_ita_Latn_8000", "xcopa_tr": "flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2", "xcopa_vi": "flexitok/reindexed-flexitok-bpe_vie_Latn_8000", "xcopa_zh": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000", "xstorycloze_ar": "flexitok/reindexed-flexitok-bpe_arb_Arab_8000", "xstorycloze_en": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "xstorycloze_es": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000", "xstorycloze_id": "flexitok/reindexed-flexitok-bpe_ind_Latn_8000", "xstorycloze_ru": "flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000", "xstorycloze_zh": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000", "paws_de": "flexitok/reindexed-flexitok-bpe_deu_Latn_8000", "paws_en": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "paws_es": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000", "paws_fr": "flexitok/reindexed-flexitok-bpe_fra_Latn_8000", "paws_ja": "flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2", "paws_zh": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000", "hellaswag_ar": "flexitok/reindexed-flexitok-bpe_arb_Arab_8000", "hellaswag_da": "flexitok/reindexed-flexitok-bpe_dan_Latn_8000", "hellaswag_de": "flexitok/reindexed-flexitok-bpe_deu_Latn_8000", "hellaswag_fr": "flexitok/reindexed-flexitok-bpe_fra_Latn_8000", "hellaswag_hu": "flexitok/reindexed-flexitok-bpe_hun_Latn_8000", "hellaswag_id": "flexitok/reindexed-flexitok-bpe_ind_Latn_8000", "hellaswag_it": "flexitok/reindexed-flexitok-bpe_ita_Latn_8000", "hellaswag_nl": "flexitok/reindexed-flexitok-bpe_nld_Latn_8000", "hellaswag_pt": "flexitok/reindexed-flexitok-bpe_por_Latn_8000", "hellaswag_ru": "flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000", "hellaswag_sv": "flexitok/reindexed-flexitok-bpe_swe_Latn_8000", "hellaswag_vi": "flexitok/reindexed-flexitok-bpe_vie_Latn_8000", "hellaswag_es": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000"}}}}