Manifest — 4.0B tokens

data/manifest.json

@@ -1,51 +1,21 @@
 {
-  "total_tokens": 23121221911,
-  "total_bytes": 46242443822,
+  "total_tokens": 4000008192,
+  "total_bytes": 8000016384,
   "format": "uint16",
   "tokenizer": "akpsahan/s8",
   "vocab_size": 3200,
   "datasets": {
-    "english_cx": {
-      "repo": "akpsahan/EnglishCX",
-      "tokens": 8000000000,
-      "budget": 8000000000,
-      "coverage": 100.0
-    },
-    "fineweb_edu": {
-      "repo": "HuggingFaceFW/fineweb-edu",
-      "tokens": 5000000000,
-      "budget": 5000000000,
-      "coverage": 100.0
-    },
-    "nemotron_math": {
-      "repo": "nvidia/Nemotron-CC-Math-v1",
-      "tokens": 0,
+    "math": {
+      "repo": "open-web-math/open-web-math",
+      "tokens": 4000000000,
       "budget": 4000000000,
-      "coverage": 0.0
-    },
-    "sinhala_cx": {
-      "repo": "akpsahan/SinhalaCX",
-      "tokens": 8000000000,
-      "budget": 8000000000,
       "coverage": 100.0
     },
-    "sinhalen_gz": {
-      "repo": "akpsahan/SinhalenGZ",
-      "tokens": 2000000000,
-      "budget": 2000000000,
+    "ot": {
+      "repo": "akpsahan/ot",
+      "tokens": 8192,
+      "budget": 8192,
       "coverage": 100.0
-    },
-    "coder_combined": {
-      "repo": "akpsahan/Coder-combined-pretraining-corpus",
-      "tokens": 74567978,
-      "budget": 2500000000,
-      "coverage": 3.0
-    },
-    "langchain_docs": {
-      "repo": "Deepak9/langchain-docs",
-      "tokens": 46653933,
-      "budget": 500000000,
-      "coverage": 9.3
     }
   }
 }