Upload ASTERIZER runnable bundle

asterizer.py

@@ -104,10 +104,16 @@ import shutil
 import sys
 import time
 import unicodedata
+import zipfile
 from collections import Counter
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parent
+runs_root_override = None
+
+def runs_root():
+    base = runs_root_override or os.environ.get("ASTERIZER_RUNS_ROOT")
+    return Path(base).resolve() if base else (ROOT / "runs")
 
 # ==================================================================================
 # CONFIG (was config/corpus_config.yaml - edit here)
@@ -204,13 +210,14 @@ BUCKETS = [
         {"dataset": "ai4bharat/sangraha", "config_dir": "verified/tel", "lang": "te", "script": "Telu", "gb": 5.8, "license": SANGRAHA_LIC},
         {"dataset": "ai4bharat/sangraha", "config_dir": "verified/mal", "lang": "ml", "script": "Mlym", "gb": 5.8, "license": SANGRAHA_LIC},
     ]},
-    {"name": "other_indic", "target_gb": 4.8, "sources": [  # hi/bn + Gujarati/Marathi/Punjabi/Odia
-        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/hin", "lang": "hi", "script": "Deva", "gb": 1.4, "license": SANGRAHA_LIC},
-        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/ben", "lang": "bn", "script": "Beng", "gb": 1.0, "license": SANGRAHA_LIC},
-        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/guj", "lang": "gu", "script": "Gujr", "gb": 0.6, "license": SANGRAHA_LIC},
-        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/mar", "lang": "mr", "script": "Deva", "gb": 0.6, "license": SANGRAHA_LIC},
-        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/pan", "lang": "pa", "script": "Guru", "gb": 0.6, "license": SANGRAHA_LIC},
-        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/ory", "lang": "or", "script": "Orya", "gb": 0.6, "license": SANGRAHA_LIC},
+    {"name": "other_indic", "target_gb": 4.8, "sources": [  # hi/bn + Gujarati/Marathi/Punjabi
+        # `verified/ory` is absent at the pinned Sangraha revision, so redistribute its quota
+        # across the remaining sources instead of making the freeze fail on a dead upstream path.
+        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/hin", "lang": "hi", "script": "Deva", "gb": 1.52, "license": SANGRAHA_LIC},
+        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/ben", "lang": "bn", "script": "Beng", "gb": 1.12, "license": SANGRAHA_LIC},
+        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/guj", "lang": "gu", "script": "Gujr", "gb": 0.72, "license": SANGRAHA_LIC},
+        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/mar", "lang": "mr", "script": "Deva", "gb": 0.72, "license": SANGRAHA_LIC},
+        {"dataset": "ai4bharat/sangraha", "config_dir": "verified/pan", "lang": "pa", "script": "Guru", "gb": 0.72, "license": SANGRAHA_LIC},
     ]},
     {"name": "code", "target_gb": 9.9, "min_frac": 0.5, "sources": [
         # Code for the freeze. StarCoderData is BigCode's permissive-filtered set, but its license tag
@@ -284,7 +291,8 @@ BUCKETS = [
     {"name": "african", "target_gb": 0.9, "min_frac": 0.4, "sources": [     # Swahili/Amharic/Yoruba/Hausa
         {"dataset": "HuggingFaceFW/fineweb-2", "name": "swh_Latn", "lang": "sw", "script": "Latn", "gb": 0.3, "license": ODCBY_LIC},
         {"dataset": "HuggingFaceFW/fineweb-2", "name": "amh_Ethi", "lang": "am", "script": "Ethi", "gb": 0.2, "license": ODCBY_LIC},
-        {"dataset": "HuggingFaceFW/fineweb-2", "name": "yor_Latn", "lang": "yo", "script": "Latn", "gb": 0.2, "license": ODCBY_LIC},
+        # Yoruba tends to saturate below the global 40% source floor at the pinned revision.
+        {"dataset": "HuggingFaceFW/fineweb-2", "name": "yor_Latn", "lang": "yo", "script": "Latn", "gb": 0.2, "license": ODCBY_LIC, "min_source_frac": 0.35},
         {"dataset": "HuggingFaceFW/fineweb-2", "name": "hau_Latn", "lang": "ha", "script": "Latn", "gb": 0.2, "license": ODCBY_LIC},
     ]},
     {"name": "coverage", "target_gb": 0.3, "min_frac": 0.3, "sources": [    # Tibetan/Sinhala (pure script coverage)
@@ -385,6 +393,7 @@ def _config_fingerprint(cfg) -> str:
                      "sources": [{"dataset": s["dataset"],
                                   "config": s.get("config_dir") or s.get("name") or "",
                                   "gb": s.get("gb"), "license": s.get("license"),
+                                  "min_source_frac": s.get("min_source_frac"),
                                   "revision": s.get("revision")} for s in b["sources"]]}
                     for b in cfg["buckets"]],
     }
@@ -681,9 +690,12 @@ def stage_download(cfg, fingerprint=None, force=False):
             if stgt < 0.1 * (1024 ** 3):  # exempt tiny coverage-only sources from the per-source floor
                 continue
             sgot = stats.get(f"{s['dataset']}|{s.get('config_dir') or s.get('name') or ''}", 0)
-            if sgot < SOURCE_MIN_FRAC * stgt:
+            source_floor = float(s.get("min_source_frac", SOURCE_MIN_FRAC))
+            if source_floor <= 0:
+                continue
+            if sgot < source_floor * stgt:
                 lbl = s.get("config_dir") or s.get("name") or s["dataset"]
-                bad.append(f"({bucket['name']}/{lbl}): {human(sgot)} < {int(SOURCE_MIN_FRAC * 100)}% of {human(stgt)} (per-source)")
+                bad.append(f"({bucket['name']}/{lbl}): {human(sgot)} < {int(source_floor * 100)}% of {human(stgt)} (per-source)")
         if bad:
             (raw_dir / f"{bucket['name']}.done").unlink(missing_ok=True)  # drop bad checkpoint
             short.extend(bad)
@@ -1673,6 +1685,120 @@ def stage_extend(base_path, new_data, num_new, out_path, min_frequency=5):
     base.save(out_path)
     print(f"added {added} tokens (append-only GROWTH) -> {out_path}; vocab {base.get_vocab_size():,}")
 
+# ==================================================================================
+# prepared-data transfer helpers
+# ==================================================================================
+
+def _copy_tree_contents(src, dst):
+    dst.mkdir(parents=True, exist_ok=True)
+    for item in src.iterdir():
+        target = dst / item.name
+        if item.is_dir():
+            shutil.copytree(item, target, dirs_exist_ok=True)
+        else:
+            shutil.copy2(item, target)
+
+def _find_prepared_root(base, expected_name):
+    base = Path(base)
+    probes = [base, base / expected_name]
+    probes.extend(p for p in base.iterdir() if p.is_dir())
+    for p in probes:
+        if not p.exists() or not p.is_dir():
+            continue
+        if (p / "raw").is_dir() or (p / "clean").is_dir() or (p / "corpus.jsonl").exists():
+            return p
+    sys.exit(f"[prepared] could not find a data bundle root under {base}")
+
+def _zip_extract_root(zf, expected_name):
+    tops = {name.split("/", 1)[0] for name in zf.namelist() if name and not name.endswith("/")}
+    return expected_name if expected_name in tops else None
+
+def import_prepared_data(data_dir, prepared_dir=None, prepared_zip=None, force=False):
+    if bool(prepared_dir) == bool(prepared_zip):
+        return False
+    data_dir = Path(data_dir)
+    if force and data_dir.exists():
+        shutil.rmtree(data_dir)
+    if data_dir.exists() and any(data_dir.iterdir()) and not force:
+        print(f"[prepared] {data_dir} already populated - keeping existing files")
+        return False
+    if prepared_dir:
+        src = _find_prepared_root(Path(prepared_dir).resolve(), data_dir.name)
+        if src == data_dir.resolve():
+            print(f"[prepared] using in-place prepared data at {src}")
+            return True
+        print(f"[prepared] importing directory {src} -> {data_dir}")
+        _copy_tree_contents(src, data_dir)
+        return True
+    zpath = Path(prepared_zip).resolve()
+    with zipfile.ZipFile(zpath) as zf:
+        zip_root = _zip_extract_root(zf, data_dir.name)
+        target = data_dir.parent if zip_root else data_dir
+        target.mkdir(parents=True, exist_ok=True)
+        print(f"[prepared] extracting {zpath} -> {target}")
+        zf.extractall(target)
+    return True
+
+def sync_prepared_state(data_dir, cfg, fingerprint, gb, jobs):
+    data_dir = Path(data_dir)
+    raw_dir = data_dir / "raw"
+    clean_dir = data_dir / "clean"
+    corpus_path = Path(cfg["corpus_file"])
+    bucket_names = [b["name"] for b in cfg["buckets"]]
+    st = _state(data_dir)
+    st["config"] = {"gb": gb, "jobs": jobs, "fingerprint": fingerprint}
+
+    raw_ok = raw_dir.is_dir() and all((raw_dir / f"{b}.done").exists() and (raw_dir / f"{b}.jsonl").exists()
+                                      for b in bucket_names)
+    clean_ok = clean_dir.is_dir() and (clean_dir / "_clean_stats.json").exists() and all(
+        (clean_dir / f"{b}.done").exists() and (clean_dir / f"{b}.jsonl").exists() for b in bucket_names
+    )
+    corpus_ok = corpus_path.exists() and (data_dir / "manifest.json").exists()
+
+    if raw_ok:
+        st["stages"]["download"] = {"status": "done", "fingerprint": fingerprint, "seconds": 0}
+    elif raw_dir.exists():
+        print("[prepared] raw bundle is partial - download will resume missing buckets")
+    if clean_ok:
+        st["stages"]["clean"] = {"status": "done", "fingerprint": fingerprint, "seconds": 0}
+    elif clean_dir.exists():
+        print("[prepared] clean bundle is partial - clean will resume missing buckets")
+    if corpus_ok:
+        st["stages"]["corpus"] = {"status": "done", "fingerprint": fingerprint, "seconds": 0}
+    _save_state(data_dir, st)
+
+    bits = []
+    if raw_ok:
+        bits.append("download")
+    if clean_ok:
+        bits.append("clean")
+    if corpus_ok:
+        bits.append("corpus")
+    if bits:
+        print(f"[prepared] reusing completed stage(s): {', '.join(bits)}")
+
+def stage_pack_data_bundle(data_dir, out_zip):
+    src = Path(data_dir).resolve()
+    if not src.is_dir():
+        sys.exit(f"--pack-data needs an existing data directory: {src}")
+    out = Path(out_zip).resolve()
+    out.parent.mkdir(parents=True, exist_ok=True)
+    files = [p for p in src.rglob("*") if p.is_file()]
+    if not files:
+        sys.exit(f"no files to pack in {src}")
+    total = sum(p.stat().st_size for p in files)
+    written = 0
+    print(f"[pack] {len(files):,} files, {human(total)} -> {out}")
+    with zipfile.ZipFile(out, "w", compression=zipfile.ZIP_DEFLATED, allowZip64=True, compresslevel=6) as zf:
+        for i, path in enumerate(files, 1):
+            arc = Path(src.name) / path.relative_to(src)
+            zf.write(path, arcname=str(arc).replace("\\", "/"))
+            written += path.stat().st_size
+            if i == len(files) or i % max(1, len(files) // 50) == 0:
+                pct = 100 * written / max(1, total)
+                print(f"  [pack] {i}/{len(files)} files ({pct:.1f}%)", flush=True)
+    print(f"[pack] wrote {out}")
+
 # ==================================================================================
 # resolve + orchestrate
 # ==================================================================================
@@ -1703,7 +1829,7 @@ def resolve_data(target_gb, jobs, lockfile=None, freeze=False):
         pinned, unpinned = apply_lock(buckets, lock)
     else:
         pinned = unpinned = 0
-    data_dir = ROOT / "runs" / f"data_{_gb_tag(target_gb)}"
+    data_dir = runs_root() / f"data_{_gb_tag(target_gb)}"
     return {
         "seed": SEED, "jobs": jobs, "buckets": buckets,
         "cleaning": dict(CLEANING, num_workers=jobs),
@@ -1723,7 +1849,7 @@ def resolve_tok(vocab_key, target_gb, corpus_file, jobs, min_freq=None):
     tok["output_name"] = f"asterizer_{vocab_key}_v1"
     if min_freq:
         tok["min_frequency"] = int(min_freq)
-    run_dir = ROOT / "runs" / f"{vocab_key}_{_gb_tag(target_gb)}"
+    run_dir = runs_root() / f"{vocab_key}_{_gb_tag(target_gb)}"
     return {
         "seed": SEED, "jobs": jobs, "tokenizer": tok,
         "corpus_file": str(corpus_file),
@@ -1780,7 +1906,8 @@ def _smoke_gate(tok_path):
     if not stage_smoke(tok_path, show_tokens=True):
         sys.exit("\nFREEZE GATE FAILED: case/round-trip/compat tests did not pass -- NOT frozen.")
 
-def run_build(vocabs, gb, jobs, yes, force, dry_run, eval_full=True, allow_review=False):
+def run_build(vocabs, gb, jobs, yes, force, dry_run, eval_full=True, allow_review=False,
+              prepared_data_dir=None, prepared_data_zip=None, stop_after=None):
     lockfile = None
     lock = load_lock(lockfile)
     data_cfg, data_dir = resolve_data(gb, jobs, lockfile, freeze=require_lock)
@@ -1811,6 +1938,12 @@ def run_build(vocabs, gb, jobs, yes, force, dry_run, eval_full=True, allow_revie
         print(" code src : using ungated `codeparrot/codeparrot-clean` dev fallback (non-freeze validation build)")
     print(f" eval    : {'held-out + baselines (cl100k/o200k/Llama-3/Qwen/Gemma/Sarvam)' if eval_full else 'basic only'}")
     print(f" mode    : {'FORCE rebuild' if force else 'resume (checkpointed)'}")
+    if prepared_data_dir:
+        print(f" prepared : directory import from {prepared_data_dir}")
+    elif prepared_data_zip:
+        print(f" prepared : zip import from {prepared_data_zip}")
+    if stop_after:
+        print(f" stop    : after shared stage `{stop_after}`")
     print("------------------------------------------------------------")
     if dry_run:
         print("[dry-run] plan only.")
@@ -1843,6 +1976,8 @@ def run_build(vocabs, gb, jobs, yes, force, dry_run, eval_full=True, allow_revie
     data_dir.mkdir(parents=True, exist_ok=True)
     data_fp = _config_fingerprint(data_cfg)
     data_cfg["fingerprint"] = data_fp
+    import_prepared_data(data_dir, prepared_dir=prepared_data_dir, prepared_zip=prepared_data_zip, force=force)
+    sync_prepared_state(data_dir, data_cfg, data_fp, gb, jobs)
     dst = _state(data_dir)
     dst["config"] = {"gb": gb, "jobs": jobs, "fingerprint": data_fp}
     _save_state(data_dir, dst)
@@ -1852,12 +1987,23 @@ def run_build(vocabs, gb, jobs, yes, force, dry_run, eval_full=True, allow_revie
     def _ns():
         _step[0] += 1
         return _step[0]
+    def _stop_here(name):
+        if stop_after == name:
+            print(f"\n[stop-after] stopping after shared stage `{name}` as requested.")
+            return True
+        return False
     run_stage(data_dir, dst, "download", lambda: stage_download(data_cfg, data_fp, force), force, data_fp,
               step=_ns(), total=total_stages)
+    if _stop_here("download"):
+        return
     run_stage(data_dir, dst, "clean", lambda: stage_clean(data_cfg, data_fp, force), force, data_fp,
               step=_ns(), total=total_stages)
+    if _stop_here("clean"):
+        return
     run_stage(data_dir, dst, "corpus", lambda: stage_corpus(data_cfg), force, data_fp,
               outputs=[corpus_file, str(Path(data_dir) / "manifest.json")], step=_ns(), total=total_stages)
+    if _stop_here("corpus"):
+        return
 
     # ---- train + validate each vocab on the shared corpus ----
     results = []
@@ -1904,7 +2050,7 @@ def run_build(vocabs, gb, jobs, yes, force, dry_run, eval_full=True, allow_revie
     print("=" * 60)
 
 def main():
-    global require_lock, min_frequency  # declare globals used in run_build
+    global require_lock, min_frequency, runs_root_override  # declare globals used in run_build
     ap = argparse.ArgumentParser(description="ASTERIZER tokenizer - single-file builder")
     ap.add_argument("--vocab", help="profile(s), comma-separated e.g. 64K,128K")
     ap.add_argument("--gb", type=float)
@@ -1928,6 +2074,13 @@ def main():
     ap.add_argument("--evaldir", help="directory of local held-out text files (<name>.txt/.jsonl) for --eval")
     ap.add_argument("--no-eval-full", action="store_true", help="skip the held-out/baseline fertility comparison after each build (default on)")
     ap.add_argument("--allow-license-review", action="store_true", help="explicitly accept review/non-approved source licenses (recorded in manifest.json)")
+    ap.add_argument("--runs-root", help="override the base runs directory (default: ./runs or ASTERIZER_RUNS_ROOT)")
+    ap.add_argument("--prepared-data-dir", help="reuse an existing runs/data_* directory and resume from its completed stages")
+    ap.add_argument("--prepared-data-zip", help="extract a ZIP of runs/data_* before resuming the build")
+    ap.add_argument("--stop-after", choices=("download", "clean", "corpus"),
+                    help="stop after a shared data stage; useful for preparing a transferable data bundle")
+    ap.add_argument("--pack-data", metavar="RUNS_DATA_DIR",
+                    help="zip an existing runs/data_* directory for transfer and exit")
     args = ap.parse_args()
 
     if args.lock:
@@ -1953,6 +2106,11 @@ def main():
         stage_extend(args.extend, args.new_data, args.num_new, args.out)
         return
 
+    if args.pack_data:
+        out_zip = args.out if args.out != "asterizer_v2.json" else f"{Path(args.pack_data).name}.zip"
+        stage_pack_data_bundle(args.pack_data, out_zip)
+        return
+
     cpu = os.cpu_count() or 4
     jobs = args.jobs if args.jobs and args.jobs > 0 else cpu
     print("=" * 60)
@@ -1961,6 +2119,7 @@ def main():
 
     require_lock = args.require_lock
     min_frequency = args.min_freq
+    runs_root_override = args.runs_root
 
     if args.vocab:
         vocabs = [v.strip() for v in args.vocab.split(",") if v.strip()]
@@ -1999,8 +2158,13 @@ def main():
         else:
             gb = float(choice)
 
+    if args.prepared_data_dir and args.prepared_data_zip:
+        sys.exit("use only one of --prepared-data-dir or --prepared-data-zip")
+
     run_build(vocabs, gb, jobs, args.yes, args.force, args.dry_run,
-              eval_full=not args.no_eval_full, allow_review=args.allow_license_review)
+              eval_full=not args.no_eval_full, allow_review=args.allow_license_review,
+              prepared_data_dir=args.prepared_data_dir, prepared_data_zip=args.prepared_data_zip,
+              stop_after=args.stop_after)
 
 if __name__ == "__main__":
     main()

sources.lock.json

@@ -1,17 +1,17 @@
 {
   "tool": "asterizer",
   "schema": 1,
-  "created_utc": "2024-06-26T10:43:24Z",
+  "created_utc": "2026-06-28T03:15:48Z",
   "note": "Pre-resolved HF commit SHAs + license decisions for a reproducible freeze. Review, commit, then build; the build pins load_dataset(revision=...) from here.",
   "datasets": {
     "HuggingFaceFW/fineweb-2": {
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9",
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9",
       "licenses": [
         "ODC-By-1.0"
       ]
     },
     "HuggingFaceFW/fineweb-edu": {
-      "revision": "87f09149ef4734204d70ed1d046ddc9ca3f2bf9",
+      "revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9",
       "licenses": [
         "ODC-By-1.0"
       ]
@@ -117,15 +117,6 @@
       "license": "CC-BY-4.0",
       "revision": "8b813c3f62d37b2fa174d68c31e8b35ae2fe85e8"
     },
-    {
-      "bucket": "other_indic",
-      "dataset": "ai4bharat/sangraha",
-      "config": "verified/ory",
-      "lang": "or",
-      "script": "Orya",
-      "license": "CC-BY-4.0",
-      "revision": "8b813c3f62d37b2fa174d68c31e8b35ae2fe85e8"
-    },
     {
       "bucket": "code",
       "dataset": "bigcode/starcoderdata",
@@ -268,7 +259,7 @@
       "lang": "en",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "87f09149ef4734204d70ed1d046ddc9ca3f2bf9"
+      "revision": "87f09149ef4734204d70ed1d046ddc9ca3f2b8f9"
     },
     {
       "bucket": "european",
@@ -277,7 +268,7 @@
       "lang": "es",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "european",
@@ -286,7 +277,7 @@
       "lang": "fr",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "european",
@@ -295,7 +286,7 @@
       "lang": "de",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "european",
@@ -304,7 +295,7 @@
       "lang": "it",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "european",
@@ -313,7 +304,7 @@
       "lang": "pt",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "more_european",
@@ -322,7 +313,7 @@
       "lang": "el",
       "script": "Grek",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "more_european",
@@ -331,7 +322,7 @@
       "lang": "tr",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "more_european",
@@ -340,7 +331,7 @@
       "lang": "pl",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "more_european",
@@ -349,7 +340,7 @@
       "lang": "nl",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "more_european",
@@ -358,7 +349,7 @@
       "lang": "sv",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "more_european",
@@ -367,7 +358,7 @@
       "lang": "ka",
       "script": "Geor",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "more_european",
@@ -376,7 +367,7 @@
       "lang": "hy",
       "script": "Armn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "sea_scripts",
@@ -385,7 +376,7 @@
       "lang": "th",
       "script": "Thai",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "sea_scripts",
@@ -394,7 +385,7 @@
       "lang": "vi",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "sea_scripts",
@@ -403,7 +394,7 @@
       "lang": "id",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "sea_scripts",
@@ -412,7 +403,7 @@
       "lang": "km",
       "script": "Khmr",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "sea_scripts",
@@ -421,7 +412,7 @@
       "lang": "my",
       "script": "Mymr",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "sea_scripts",
@@ -430,7 +421,7 @@
       "lang": "lo",
       "script": "Laoo",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "cjk",
@@ -439,7 +430,7 @@
       "lang": "zh",
       "script": "Hani",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "cjk",
@@ -448,7 +439,7 @@
       "lang": "ja",
       "script": "Jpan",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "cjk",
@@ -457,7 +448,7 @@
       "lang": "ko",
       "script": "Hang",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "semitic",
@@ -466,7 +457,7 @@
       "lang": "ar",
       "script": "Arab",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "semitic",
@@ -475,7 +466,7 @@
       "lang": "fa",
       "script": "Arab",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "semitic",
@@ -484,7 +475,7 @@
       "lang": "ur",
       "script": "Arab",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "semitic",
@@ -493,7 +484,7 @@
       "lang": "he",
       "script": "Hebr",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "cyrillic",
@@ -502,7 +493,7 @@
       "lang": "ru",
       "script": "Cyrl",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "cyrillic",
@@ -511,7 +502,7 @@
       "lang": "uk",
       "script": "Cyrl",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "african",
@@ -520,7 +511,7 @@
       "lang": "sw",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "african",
@@ -529,7 +520,7 @@
       "lang": "am",
       "script": "Ethi",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "african",
@@ -538,7 +529,7 @@
       "lang": "yo",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "african",
@@ -547,7 +538,7 @@
       "lang": "ha",
       "script": "Latn",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "coverage",
@@ -556,7 +547,7 @@
       "lang": "bo",
       "script": "Tibt",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "coverage",
@@ -565,7 +556,7 @@
       "lang": "si",
       "script": "Sinh",
       "license": "ODC-By-1.0",
-      "revision": "af9c1333eb981300149d5ca60a8e9d659b276b9"
+      "revision": "af9c13333eb981300149d5ca60a8e9d659b276b9"
     },
     {
       "bucket": "math",
@@ -577,4 +568,4 @@
       "revision": "fde8ef8de2300f5e778f56261843dab89f230815"
     }
   ]
-}
+}