Upload recipe/corpus_stats.json with huggingface_hub

recipe/corpus_stats.json

@@ -0,0 +1,32 @@
+{
+  "recipe_version": "hf-bytelevel-bpe-v1",
+  "test_run": true,
+  "weights_target": {
+    "exotic": 0.7,
+    "eng": 0.1,
+    "spa": 0.1,
+    "code": 0.1
+  },
+  "weights_actual": {
+    "modern": 0.4962163572422402,
+    "old": 0.20383415129525775,
+    "eng": 0.09999221223436393,
+    "spa": 0.09995951001521453,
+    "code": 0.09999776921292355,
+    "exotic": 0.700050508537498
+  },
+  "bytes": {
+    "modern": 24824308,
+    "old": 10197249,
+    "eng": 5002329,
+    "spa": 5000693,
+    "code": 5002607,
+    "exotic": 35021557
+  },
+  "total_bytes": 50027186,
+  "vocab_target": 256000,
+  "min_frequency": 2,
+  "special_token_count": 94,
+  "language_token_count": 60,
+  "external_to_exotic_ratio": 0.14285714285714288
+}