checkpoints/safe_v2_2nodes/0000005000/train_state_00007.json

{"step": 5000, "acc_step": 0, "data_loader_state": {"it_state": {"start_token": 1140, "it_state": {"it_state": {"root_dir": "/scratch/craffel/lingua/data/flexitok/", "sources": {"fw_edu": 0.4, "dan_Latn": 0.0216582869670702, "swe_Latn": 0.0216359765418466, "vie_Latn": 0.0197485510268674, "hun_Latn": 0.0247194573562308, "fas_Arab": 0.0205634624231076, "tur_Latn": 0.0235455794841729, "ces_Latn": 0.0248024455266208, "arb_Arab": 0.0234323706569333, "ell_Grek": 0.0233670886888026, "ind_Latn": 0.0269322054593488, "nld_Latn": 0.0277796326621489, "pol_Latn": 0.0294120104572311, "por_Latn": 0.0301413168306825, "ita_Latn": 0.0324056371021865, "jpn_Jpan": 0.03553104151369, "fra_Latn": 0.0381835560678536, "spa_Latn": 0.0387222793083669, "deu_Latn": 0.0419925340453022, "cmn_Hani": 0.0454067521384114, "rus_Cyrl": 0.0500198157431261}, "source_to_state": {"fw_edu": {"file_path": "/scratch/craffel/lingua/data/flexitok/fw_edu/fineweb_edu_100bt.chunk.03.jsonl", "position": 1143857600, "block_size": 2, "offset": 1, "current_iter": 0}, "dan_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/dan_Latn/fineweb_2_hq.dan_Latn.chunk.03.jsonl", "position": 50348968, "block_size": 2, "offset": 1, "current_iter": 0}, "swe_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/swe_Latn/fineweb_2_hq.swe_Latn.chunk.03.jsonl", "position": 48498725, "block_size": 2, "offset": 1, "current_iter": 0}, "vie_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/vie_Latn/fineweb_2_hq.vie_Latn.chunk.03.jsonl", "position": 50609378, "block_size": 2, "offset": 1, "current_iter": 0}, "hun_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/hun_Latn/fineweb_2_hq.hun_Latn.chunk.03.jsonl", "position": 71042274, "block_size": 2, "offset": 1, "current_iter": 0}, "fas_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/fas_Arab/fineweb_2_hq.fas_Arab.chunk.03.jsonl", "position": 56222798, "block_size": 2, "offset": 1, "current_iter": 0}, "tur_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/tur_Latn/fineweb_2_hq.tur_Latn.chunk.03.jsonl", "position": 45072781, "block_size": 2, "offset": 1, "current_iter": 0}, "ces_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ces_Latn/fineweb_2_hq.ces_Latn.chunk.03.jsonl", "position": 68777465, "block_size": 2, "offset": 1, "current_iter": 0}, "arb_Arab": {"file_path": "/scratch/craffel/lingua/data/flexitok/arb_Arab/fineweb_2_hq.arb_Arab.chunk.03.jsonl", "position": 76731180, "block_size": 2, "offset": 1, "current_iter": 0}, "ell_Grek": {"file_path": "/scratch/craffel/lingua/data/flexitok/ell_Grek/fineweb_2_hq.ell_Grek.chunk.03.jsonl", "position": 96184431, "block_size": 2, "offset": 1, "current_iter": 0}, "ind_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ind_Latn/fineweb_2_hq.ind_Latn.chunk.03.jsonl", "position": 65460224, "block_size": 2, "offset": 1, "current_iter": 0}, "nld_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/nld_Latn/fineweb_2_hq.nld_Latn.chunk.03.jsonl", "position": 55620029, "block_size": 2, "offset": 1, "current_iter": 0}, "pol_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/pol_Latn/fineweb_2_hq.pol_Latn.chunk.03.jsonl", "position": 56484583, "block_size": 2, "offset": 1, "current_iter": 0}, "por_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/por_Latn/fineweb_2_hq.por_Latn.chunk.03.jsonl", "position": 57048765, "block_size": 2, "offset": 1, "current_iter": 0}, "ita_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/ita_Latn/fineweb_2_hq.ita_Latn.chunk.03.jsonl", "position": 67248769, "block_size": 2, "offset": 1, "current_iter": 0}, "jpn_Jpan": {"file_path": "/scratch/craffel/lingua/data/flexitok/jpn_Jpan/fineweb_2_hq.jpn_Jpan.chunk.03.jsonl", "position": 67632154, "block_size": 2, "offset": 1, "current_iter": 0}, "fra_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/fra_Latn/fineweb_2_hq.fra_Latn.chunk.03.jsonl", "position": 93130927, "block_size": 2, "offset": 1, "current_iter": 0}, "spa_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/spa_Latn/fineweb_2_hq.spa_Latn.chunk.03.jsonl", "position": 82280986, "block_size": 2, "offset": 1, "current_iter": 0}, "deu_Latn": {"file_path": "/scratch/craffel/lingua/data/flexitok/deu_Latn/fineweb_2_hq.deu_Latn.chunk.03.jsonl", "position": 102047966, "block_size": 2, "offset": 1, "current_iter": 0}, "cmn_Hani": {"file_path": "/scratch/craffel/lingua/data/flexitok/cmn_Hani/fineweb_2_hq.cmn_Hani.chunk.03.jsonl", "position": 102801555, "block_size": 2, "offset": 1, "current_iter": 0}, "rus_Cyrl": {"file_path": "/scratch/craffel/lingua/data/flexitok/rus_Cyrl/fineweb_2_hq.rus_Cyrl.chunk.03.jsonl", "position": 251106082, "block_size": 2, "offset": 1, "current_iter": 0}}, "rng_state": {"bit_generator": "PCG64", "state": {"state": 11210963299561688403176053937219378156, "inc": 111269986385306254579901734703632045793}, "has_uint32": 0, "uinteger": 0}}, "add_bos": true, "add_eos": true, "name": "supertokenizer", "path": "meta-llama/Llama-3.2-1B", "tokenizers": [{"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_arb_Arab_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ces_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000_v2", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_dan_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_deu_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ell_Grek_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_fas_Arab_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_fra_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_hun_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ind_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ita_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_nld_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_pol_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_por_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_swe_Latn_8000", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2", "load_supermapping": false}, {"name": "huggingface", "path": "flexitok/reindexed-flexitok-bpe_vie_Latn_8000", "load_supermapping": false}], "dropout": 0.0, "rng_state": {"bit_generator": "PCG64", "state": {"state": 31543473163452429948919053486845826939, "inc": 332724090758049132448979897138935081983}, "has_uint32": 1, "uinteger": 2753492566}, "seed": 42, "superset_code_name": "safe_v2", "n_words": 122554, "routing": {"source_to_tokenizer": {"arb_Arab": "flexitok/reindexed-flexitok-bpe_arb_Arab_8000", "ces_Latn": "flexitok/reindexed-flexitok-bpe_ces_Latn_8000", "cmn_Hani": "flexitok/reindexed-flexitok-bpe_ltr_cmn_Hani_8000", "dan_Latn": "flexitok/reindexed-flexitok-bpe_dan_Latn_8000", "deu_Latn": "flexitok/reindexed-flexitok-bpe_deu_Latn_8000", "ell_Grek": "flexitok/reindexed-flexitok-bpe_ell_Grek_8000", "fas_Arab": "flexitok/reindexed-flexitok-bpe_fas_Arab_8000", "fra_Latn": "flexitok/reindexed-flexitok-bpe_fra_Latn_8000", "fw_edu": "flexitok/reindexed-flexitok-bpe_fw_edu_8000", "hun_Latn": "flexitok/reindexed-flexitok-bpe_hun_Latn_8000", "ind_Latn": "flexitok/reindexed-flexitok-bpe_ind_Latn_8000", "ita_Latn": "flexitok/reindexed-flexitok-bpe_ita_Latn_8000", "jpn_Jpan": "flexitok/reindexed-flexitok-bpe_ltr_jpn_Jpan_8000_v2", "nld_Latn": "flexitok/reindexed-flexitok-bpe_nld_Latn_8000", "pol_Latn": "flexitok/reindexed-flexitok-bpe_pol_Latn_8000", "por_Latn": "flexitok/reindexed-flexitok-bpe_por_Latn_8000", "rus_Cyrl": "flexitok/reindexed-flexitok-bpe_rus_Cyrl_8000", "spa_Latn": "flexitok/reindexed-flexitok-bpe_spa_Latn_8000", "swe_Latn": "flexitok/reindexed-flexitok-bpe_swe_Latn_8000", "tur_Latn": "flexitok/reindexed-flexitok-bpe_ltr_tur_Latn_8000_v2", "vie_Latn": "flexitok/reindexed-flexitok-bpe_vie_Latn_8000"}, "suitable_tokenizer_probability": 1.0}}, "output_seq_len": 4096, "n_views": 2}, "seq_idx": 544, "rng_state": {"bit_generator": "PCG64", "state": {"state": 193556514805831417869920483681437837117, "inc": 264511291084603256659347431268637710261}, "has_uint32": 1, "uinteger": 2474072747}, "batch_size": 4, "prefetch_size": 1024}, "scheduler": {"base_lrs": [0.001], "last_epoch": 5000, "verbose": false, "_step_count": 5001, "_get_lr_called_within_step": false, "_last_lr": [0.0009976895587850427], "lr_lambdas": [{}]}}