Create app.py

app.py

@@ -0,0 +1,14 @@
+import pyarrow.parquet as pq
+import pyarrow as pa
+from pathlib import Path
+
+OUT_DIR = "/data/image-shards"
+
+for shard in Path(OUT_DIR).glob("*.parquet"):
+    table = pq.read_table(shard)
+    df = table.to_pandas()
+    
+    df["image"] = df["image"].apply(lambda b: {"bytes": b, "path": None})
+    
+    pq.write_table(pa.Table.from_pandas(df), shard, compression="snappy")
+    print(f"Fixed {shard.name}")