Update app.py

app.py

@@ -1,11 +1,9 @@
 import os
-import json
 import gc
 from pathlib import Path
 
-TOK_DIR  = "/data/tokenized"
-OUT_DIR  = "/data/ready"
-INDEX    = "/data/index.json"
+TOK_DIR = "/data/tokenized"
+OUT_DIR = "/data/ready"
 
 CRAWL_MAP = {
     "2025-05": "fw-00001-of-00006.bin",
@@ -17,9 +15,10 @@ CRAWL_MAP = {
 }
 
 os.makedirs(OUT_DIR, exist_ok=True)
-
 print("Starting merge...")
-print(f"TOK_DIR contents: {list(Path(TOK_DIR).iterdir())[:5]}")
+print(f"TOK_DIR sample: {list(Path(TOK_DIR).iterdir())[:5]}")
+
+index = {}
 
 for crawl_id, out_name in CRAWL_MAP.items():
     out_path = Path(OUT_DIR) / out_name
@@ -28,22 +27,23 @@ for crawl_id, out_name in CRAWL_MAP.items():
         continue
 
     shards = sorted(Path(TOK_DIR).glob(f"cc{crawl_id}_*.bin"))
-    print(f"  {crawl_id}: found {len(shards)} shards")
+    print(f"  {crawl_id}: {len(shards)} shards")
     if not shards:
         continue
 
     total_tokens = 0
     with open(out_path, "wb") as out_f:
         for shard in shards:
-            with open(shard, "rb") as in_f:
-                while True:
-                    chunk = in_f.read(4096)
-                    if not chunk:
-                        break
-                    out_f.write(chunk)
-                    total_tokens += len(chunk) // 2
+            data = shard.read_bytes()
+            out_f.write(data)
+            total_tokens += len(data) // 2
+            shard.unlink()
+            del data
+            gc.collect()
             print(f"    ✓ {shard.name}")
 
+    index[out_name] = {"crawl": crawl_id, "total_tokens": total_tokens}
     print(f"  ✓ {out_name} | {total_tokens:,} tokens")
 
+Path("/data/index.json").write_text(__import__("json").dumps(index, indent=2))
 print("Done")