Upload folder using huggingface_hub

EVA_1.4B_CLM/config.json

@@ -0,0 +1,27 @@
+{
+  "vocab_size": 114,
+  "hidden_size": 1024,
+  "num_hidden_layers": 26,
+  "num_attention_heads": 16,
+  "num_key_value_heads": 16,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 8192,
+  "rms_norm_eps": "1e-6",
+  "initializer_range": 0.02,
+  "use_cache": true,
+  "num_experts": 8,
+  "num_experts_per_tok": 2,
+  "expert_capacity_factor": 1.5,
+  "router_aux_loss_coef": 0.01,
+  "moe_implementation": "megablocks",
+  "moe_world_size": 1,
+  "resid_dropout": 0.0,
+  "hidden_dropout": 0.0,
+  "dropout_warmup_steps": 0,
+  "dropout_ramp_steps": 0,
+  "dropout_schedule": "linear",
+  "attention_dropout": 0.0,
+  "gradient_clip_norm": 0.0,
+  "label_smoothing": 0.0,
+  "eos_loss_weight": 1.0
+}

EVA_1.4B_CLM/eval_results_stage1.json

@@ -0,0 +1,12 @@
+{
+  "loss": 1.2702110850016999,
+  "perplexity": 3.5616042613983154,
+  "total_tokens": 19854467,
+  "total_samples": 10112,
+  "eval_time": 791.5355548858643,
+  "eval_mode": "generation",
+  "stage": "stage1",
+  "eval_dataset": "/rna-multiverse/data/cluster/sampling/training/train_30M_sqrt_s42_50only_new.fa",
+  "batch_size": 128,
+  "num_batches": 79
+}

EVA_1.4B_CLM/model_weights.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:323c13d571d0be87e450cb7b103bf8b28396e55e877519e2d2fb3c434875d420
+size 3057231170

EVA_1.4B_CLM/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "pad_token": "<pad>",
+  "bos_token": "<bos>",
+  "eos_token": "<eos>",
+  "unk_token": "<unk>"
+}

EVA_1.4B_CLM/tokenizer.json

@@ -0,0 +1,1171 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": {
+    "strategy": "BatchLongest",
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 0,
+    "pad_type_id": 0,
+    "pad_token": "<pad>"
+  },
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<bos>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<eos>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<bos_glm>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "<eos_span>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 5,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 6,
+      "content": "<span_0>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 7,
+      "content": "<span_1>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 8,
+      "content": "<span_2>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 9,
+      "content": "<span_3>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 10,
+      "content": "<span_4>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 11,
+      "content": "<span_5>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 12,
+      "content": "<span_6>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 13,
+      "content": "<span_7>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 14,
+      "content": "<span_8>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 15,
+      "content": "<span_9>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 16,
+      "content": "<span_10>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 17,
+      "content": "<span_11>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 18,
+      "content": "<span_12>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 19,
+      "content": "<span_13>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 20,
+      "content": "<span_14>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 21,
+      "content": "<span_15>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 22,
+      "content": "<span_16>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 23,
+      "content": "<span_17>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 24,
+      "content": "<span_18>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 25,
+      "content": "<span_19>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 26,
+      "content": "<span_20>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 27,
+      "content": "<span_21>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 28,
+      "content": "<span_22>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 29,
+      "content": "<span_23>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 30,
+      "content": "<span_24>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 31,
+      "content": "<span_25>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 32,
+      "content": "<span_26>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 33,
+      "content": "<span_27>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 34,
+      "content": "<span_28>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 35,
+      "content": "<span_29>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 36,
+      "content": "<span_30>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 37,
+      "content": "<span_31>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 38,
+      "content": "<span_32>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 39,
+      "content": "<span_33>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 40,
+      "content": "<span_34>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 41,
+      "content": "<span_35>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 42,
+      "content": "<span_36>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 43,
+      "content": "<span_37>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 44,
+      "content": "<span_38>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 45,
+      "content": "<span_39>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 46,
+      "content": "<span_40>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 47,
+      "content": "<span_41>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 48,
+      "content": "<span_42>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 49,
+      "content": "<span_43>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 50,
+      "content": "<span_44>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 51,
+      "content": "<span_45>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 52,
+      "content": "<span_46>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 53,
+      "content": "<span_47>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 54,
+      "content": "<span_48>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 55,
+      "content": "<span_49>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 56,
+      "content": "<rna_mRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 57,
+      "content": "<rna_rRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 58,
+      "content": "<rna_tRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 59,
+      "content": "<rna_sRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 60,
+      "content": "<rna_lncRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 61,
+      "content": "<rna_circRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 62,
+      "content": "<rna_viral_RNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 63,
+      "content": "<rna_miRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 64,
+      "content": "<rna_snoRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 65,
+      "content": "<rna_snRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 66,
+      "content": "<rna_piRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 67,
+      "content": "<rna_ribozyme>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 68,
+      "content": "<rna_scaRNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 69,
+      "content": "<rna_Y_RNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 70,
+      "content": "<rna_vault_RNA>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 71,
+      "content": "d__",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 72,
+      "content": "p__",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 73,
+      "content": "c__",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 74,
+      "content": "o__",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 75,
+      "content": "f__",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 76,
+      "content": "g__",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 77,
+      "content": "s__",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 78,
+      "content": ";",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 79,
+      "content": "|",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 80,
+      "content": "_",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 81,
+      "content": "-",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 82,
+      "content": "A",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 83,
+      "content": "U",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 84,
+      "content": "G",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 85,
+      "content": "C",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 86,
+      "content": "5",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 87,
+      "content": "3",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 88,
+      "content": "a",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 89,
+      "content": "b",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 90,
+      "content": "c",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 91,
+      "content": "d",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 92,
+      "content": "e",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 93,
+      "content": "f",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 94,
+      "content": "g",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 95,
+      "content": "h",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 96,
+      "content": "i",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 97,
+      "content": "j",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 98,
+      "content": "k",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 99,
+      "content": "l",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 100,
+      "content": "m",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 101,
+      "content": "n",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 102,
+      "content": "o",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 103,
+      "content": "p",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 104,
+      "content": "q",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 105,
+      "content": "r",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 106,
+      "content": "s",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 107,
+      "content": "t",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 108,
+      "content": "u",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 109,
+      "content": "v",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 110,
+      "content": "w",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 111,
+      "content": "x",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 112,
+      "content": "y",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 113,
+      "content": "z",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": null,
+  "post_processor": null,
+  "decoder": null,
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "<unk>",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<pad>": 0,
+      "<bos>": 1,
+      "<eos>": 2,
+      "<bos_glm>": 3,
+      "<eos_span>": 4,
+      "<unk>": 5,
+      "<span_0>": 6,
+      "<span_1>": 7,
+      "<span_2>": 8,
+      "<span_3>": 9,
+      "<span_4>": 10,
+      "<span_5>": 11,
+      "<span_6>": 12,
+      "<span_7>": 13,
+      "<span_8>": 14,
+      "<span_9>": 15,
+      "<span_10>": 16,
+      "<span_11>": 17,
+      "<span_12>": 18,
+      "<span_13>": 19,
+      "<span_14>": 20,
+      "<span_15>": 21,
+      "<span_16>": 22,
+      "<span_17>": 23,
+      "<span_18>": 24,
+      "<span_19>": 25,
+      "<span_20>": 26,
+      "<span_21>": 27,
+      "<span_22>": 28,
+      "<span_23>": 29,
+      "<span_24>": 30,
+      "<span_25>": 31,
+      "<span_26>": 32,
+      "<span_27>": 33,
+      "<span_28>": 34,
+      "<span_29>": 35,
+      "<span_30>": 36,
+      "<span_31>": 37,
+      "<span_32>": 38,
+      "<span_33>": 39,
+      "<span_34>": 40,
+      "<span_35>": 41,
+      "<span_36>": 42,
+      "<span_37>": 43,
+      "<span_38>": 44,
+      "<span_39>": 45,
+      "<span_40>": 46,
+      "<span_41>": 47,
+      "<span_42>": 48,
+      "<span_43>": 49,
+      "<span_44>": 50,
+      "<span_45>": 51,
+      "<span_46>": 52,
+      "<span_47>": 53,
+      "<span_48>": 54,
+      "<span_49>": 55,
+      "<rna_mRNA>": 56,
+      "<rna_rRNA>": 57,
+      "<rna_tRNA>": 58,
+      "<rna_sRNA>": 59,
+      "<rna_lncRNA>": 60,
+      "<rna_circRNA>": 61,
+      "<rna_viral_RNA>": 62,
+      "<rna_miRNA>": 63,
+      "<rna_snoRNA>": 64,
+      "<rna_snRNA>": 65,
+      "<rna_piRNA>": 66,
+      "<rna_ribozyme>": 67,
+      "<rna_scaRNA>": 68,
+      "<rna_Y_RNA>": 69,
+      "<rna_vault_RNA>": 70,
+      "d__": 71,
+      "p__": 72,
+      "c__": 73,
+      "o__": 74,
+      "f__": 75,
+      "g__": 76,
+      "s__": 77,
+      ";": 78,
+      "|": 79,
+      "_": 80,
+      "-": 81,
+      "A": 82,
+      "U": 83,
+      "G": 84,
+      "C": 85,
+      "5": 86,
+      "3": 87,
+      "a": 88,
+      "b": 89,
+      "c": 90,
+      "d": 91,
+      "e": 92,
+      "f": 93,
+      "g": 94,
+      "h": 95,
+      "i": 96,
+      "j": 97,
+      "k": 98,
+      "l": 99,
+      "m": 100,
+      "n": 101,
+      "o": 102,
+      "p": 103,
+      "q": 104,
+      "r": 105,
+      "s": 106,
+      "t": 107,
+      "u": 108,
+      "v": 109,
+      "w": 110,
+      "x": 111,
+      "y": 112,
+      "z": 113
+    },
+    "merges": []
+  }
+}

EVA_1.4B_CLM/tokenizer_config.json

@@ -0,0 +1,114 @@
+{
+  "tokenizer_class": "LineageRNATokenizer",
+  "auto_map": {
+    "AutoTokenizer": [
+      "lineage_tokenizer.py",
+      "LineageRNATokenizer"
+    ]
+  },
+  "vocab_size": 114,
+  "pad_token": "<pad>",
+  "bos_token": "<bos>",
+  "eos_token": "<eos>",
+  "unk_token": "<unk>",
+  "rna_bases": [
+    "A",
+    "U",
+    "G",
+    "C"
+  ],
+  "special_tokens": [
+    "<pad>",
+    "<bos>",
+    "<eos>",
+    "<bos_glm>",
+    "<eos_span>",
+    "<unk>"
+  ],
+  "glm_span_tokens": [
+    "<span_0>",
+    "<span_1>",
+    "<span_2>",
+    "<span_3>",
+    "<span_4>",
+    "<span_5>",
+    "<span_6>",
+    "<span_7>",
+    "<span_8>",
+    "<span_9>",
+    "<span_10>",
+    "<span_11>",
+    "<span_12>",
+    "<span_13>",
+    "<span_14>",
+    "<span_15>",
+    "<span_16>",
+    "<span_17>",
+    "<span_18>",
+    "<span_19>",
+    "<span_20>",
+    "<span_21>",
+    "<span_22>",
+    "<span_23>",
+    "<span_24>",
+    "<span_25>",
+    "<span_26>",
+    "<span_27>",
+    "<span_28>",
+    "<span_29>",
+    "<span_30>",
+    "<span_31>",
+    "<span_32>",
+    "<span_33>",
+    "<span_34>",
+    "<span_35>",
+    "<span_36>",
+    "<span_37>",
+    "<span_38>",
+    "<span_39>",
+    "<span_40>",
+    "<span_41>",
+    "<span_42>",
+    "<span_43>",
+    "<span_44>",
+    "<span_45>",
+    "<span_46>",
+    "<span_47>",
+    "<span_48>",
+    "<span_49>"
+  ],
+  "rna_type_tokens": [
+    "<rna_mRNA>",
+    "<rna_rRNA>",
+    "<rna_tRNA>",
+    "<rna_sRNA>",
+    "<rna_lncRNA>",
+    "<rna_circRNA>",
+    "<rna_viral_RNA>",
+    "<rna_miRNA>",
+    "<rna_snoRNA>",
+    "<rna_snRNA>",
+    "<rna_piRNA>",
+    "<rna_ribozyme>",
+    "<rna_scaRNA>",
+    "<rna_Y_RNA>",
+    "<rna_vault_RNA>"
+  ],
+  "lineage_level_tokens": [
+    "d__",
+    "p__",
+    "c__",
+    "o__",
+    "f__",
+    "g__",
+    "s__"
+  ],
+  "lineage_special_chars": [
+    ";",
+    "|",
+    "_",
+    "-"
+  ],
+  "mode": "lineage",
+  "description": "Lineage-based tokenizer without species tokens or task tokens"
+}

EVA_1.4B_CLM/training_config.yaml

@@ -0,0 +1,104 @@
+data_config:
+  cache_size: 10000
+  dataset: train_30M_sqrt_s42
+  enable_reverse_augmentation: true
+  force_rebuild_index: false
+  glm_probability: 0.333
+  lineage_file: /rna-multiverse/data/training_data/lineage_greengenes.tsv
+  max_samples: null
+  max_seq_length: 8192
+  mode: mixed
+  pretrain_ratio: 0.2
+  span_config:
+    allow_overlap: false
+    coverage_probs:
+    - 0.28
+    - 0.3
+    - 0.28
+    - 0.14
+    distribution_probs:
+    - 0.3
+    - 0.5
+    - 0.2
+    max_coverage_ratios:
+    - 0.15
+    - 0.25
+    - 0.5
+    - 0.8
+    max_num_spans: 10
+    span_distributions:
+    - - 10
+      - 5
+    - - 20
+      - 10
+    - - 50
+      - 20
+  train_file: /rna-multiverse/data/cluster/sampling/training/train_30M_sqrt_s42_50only_new.fa
+  use_chunked: true
+  use_direction_tokens: true
+  use_lineage_prefix: true
+  use_rna_type_prefix: true
+distributed_config:
+  backend: nccl
+  data_parallel_size: 4
+  expert_parallel_size: 4
+  weight_parallel_size: 1
+logging_config:
+  enable_wandb: true
+  log_dir: /rna-multiverse/results/mid_training/mid_training_v4_from_v24_1.4B_checkpoint_15000_20251202/logs
+  wandb_project: rna-mid-training
+  wandb_run_name: mid_training_v4_from_v24_1.4B_checkpoint_15000
+memory_config:
+  cleanup_frequency: 100
+  enable_monitoring: true
+  gc_frequency: 50
+model_config:
+  attention_dropout: 0.0
+  dropout_ramp_steps: 0
+  dropout_schedule: linear
+  dropout_warmup_steps: 0
+  eos_loss_weight: 1.0
+  expert_capacity_factor: 1.5
+  gradient_clip_norm: 0.0
+  hidden_dropout: 0.0
+  hidden_size: 1024
+  initializer_range: 0.02
+  intermediate_size: 3072
+  label_smoothing: 0.0
+  max_position_embeddings: 8192
+  moe_implementation: megablocks
+  moe_world_size: 4
+  num_attention_heads: 16
+  num_experts: 8
+  num_experts_per_tok: 2
+  num_hidden_layers: 26
+  num_key_value_heads: 16
+  resid_dropout: 0.0
+  rms_norm_eps: 1e-6
+  router_aux_loss_coef: 0.01
+  use_cache: true
+  vocab_size: 114
+training_config:
+  adam_beta1: 0.9
+  adam_beta2: 0.95
+  adam_epsilon: 1e-8
+  bf16: true
+  dataloader_drop_last: true
+  dataloader_num_workers: 8
+  dataloader_pin_memory: true
+  fp16: false
+  gradient_accumulation_steps: 16
+  gradient_checkpointing: false
+  learning_rate: 0.0001
+  logging_steps: 30
+  max_epochs: 1
+  max_wall_time_hours: 100
+  min_lr_ratio: 0.1
+  output_dir: /rna-multiverse/results/mid_training/mid_training_v4_from_v24_1.4B_checkpoint_15000_20251202
+  per_device_train_batch_size: 4
+  resume_from_pretrain: /rna-multiverse/results/experiments/scaling_1.4B_v24_mixed_glm_stage1_20251126/checkpoint-15000
+  run_name: mid_training_v4_from_v24_1.4B_checkpoint_15000
+  save_steps: 500
+  seed: 42
+  warmup_steps: 1000
+  weight_decay: 5.0e-06

EVA_1.4B_CLM/vocab.json

@@ -0,0 +1,116 @@
+{
+  "<rna_sRNA>": 59,
+  "s__": 77,
+  "<span_18>": 24,
+  "<span_8>": 14,
+  "n": 101,
+  ";": 78,
+  "A": 82,
+  "j": 97,
+  "<rna_ribozyme>": 67,
+  "<span_21>": 27,
+  "c__": 73,
+  "d__": 71,
+  "<bos_glm>": 3,
+  "h": 95,
+  "<rna_lncRNA>": 60,
+  "d": 91,
+  "<rna_tRNA>": 58,
+  "q": 104,
+  "<span_17>": 23,
+  "<span_35>": 41,
+  "o": 102,
+  "g": 94,
+  "<span_1>": 7,
+  "<span_29>": 35,
+  "c": 90,
+  "<span_36>": 42,
+  "p__": 72,
+  "<span_49>": 55,
+  "<rna_rRNA>": 57,
+  "<rna_mRNA>": 56,
+  "x": 111,
+  "<span_23>": 29,
+  "<span_2>": 8,
+  "<span_46>": 52,
+  "<span_31>": 37,
+  "<span_40>": 46,
+  "<span_27>": 33,
+  "<span_38>": 44,
+  "<span_25>": 31,
+  "<rna_snoRNA>": 64,
+  "<span_11>": 17,
+  "<span_14>": 20,
+  "G": 84,
+  "<span_39>": 45,
+  "w": 110,
+  "<rna_circRNA>": 61,
+  "<span_13>": 19,
+  "<rna_snRNA>": 65,
+  "<rna_vault_RNA>": 70,
+  "e": 92,
+  "v": 109,
+  "<pad>": 0,
+  "<span_22>": 28,
+  "a": 88,
+  "3": 87,
+  "C": 85,
+  "r": 105,
+  "<span_43>": 49,
+  "<span_24>": 30,
+  "f": 93,
+  "<span_7>": 13,
+  "<span_44>": 50,
+  "<span_26>": 32,
+  "<span_0>": 6,
+  "t": 107,
+  "_": 80,
+  "l": 99,
+  "<span_47>": 53,
+  "5": 86,
+  "<unk>": 5,
+  "U": 83,
+  "<span_41>": 47,
+  "<eos_span>": 4,
+  "p": 103,
+  "<span_42>": 48,
+  "k": 98,
+  "<span_19>": 25,
+  "<span_6>": 12,
+  "<span_15>": 21,
+  "<span_28>": 34,
+  "<span_3>": 9,
+  "<span_33>": 39,
+  "<span_12>": 18,
+  "|": 79,
+  "b": 89,
+  "<bos>": 1,
+  "<span_5>": 11,
+  "<rna_viral_RNA>": 62,
+  "<rna_scaRNA>": 68,
+  "<rna_piRNA>": 66,
+  "u": 108,
+  "o__": 74,
+  "g__": 76,
+  "<span_48>": 54,
+  "<rna_Y_RNA>": 69,
+  "i": 96,
+  "<span_4>": 10,
+  "f__": 75,
+  "<span_30>": 36,
+  "y": 112,
+  "<span_34>": 40,
+  "z": 113,
+  "<span_45>": 51,
+  "<rna_miRNA>": 63,
+  "<span_37>": 43,
+  "<span_10>": 16,
+  "s": 106,
+  "<span_16>": 22,
+  "<span_9>": 15,
+  "<span_20>": 26,
+  "<span_32>": 38,
+  "-": 81,
+  "<eos>": 2,
+  "m": 100
+}