Buckets:
| import zipfile,json,hashlib,io | |
| from pathlib import Path | |
| from PIL import Image,ImageOps | |
| root=Path('/marimo/shuiwu'); out=root/'data/1_shuiwu' | |
| summary={'images':0,'duplicates':0,'invalid':0,'min_side':None,'max_side':0,'original_captions_ignored':0} | |
| seen=set(); dims=[] | |
| with zipfile.ZipFile(root/'downloads/dataset.zip') as z: | |
| for member in z.infolist(): | |
| if member.is_dir():continue | |
| ext=Path(member.filename).suffix.lower() | |
| if ext=='.txt':summary['original_captions_ignored']+=1;continue | |
| if ext not in {'.png','.jpg','.jpeg','.webp','.bmp','.avif'}:continue | |
| raw=z.read(member);digest=hashlib.sha256(raw).hexdigest() | |
| if digest in seen:summary['duplicates']+=1;continue | |
| try: | |
| with Image.open(io.BytesIO(raw)) as im: | |
| im=ImageOps.exif_transpose(im).convert('RGB');im.load();w,h=im.size | |
| if min(w,h)<32 or max(w,h)/min(w,h)>8:summary['invalid']+=1;continue | |
| seen.add(digest);summary['images']+=1;dims.append((w,h)) | |
| im.save(out/('{:06d}.png'.format(summary['images']))) | |
| except Exception:summary['invalid']+=1 | |
| summary['min_side']=min(min(d) for d in dims) if dims else None | |
| summary['max_side']=max(max(d) for d in dims) if dims else None | |
| summary['median_pixels']=sorted(w*h for w,h in dims)[len(dims)//2] if dims else None | |
| (root/'dataset-summary.json').write_text(json.dumps(summary,indent=2));print(json.dumps(summary)) | |
Xet Storage Details
- Size:
- 1.35 kB
- Xet hash:
- cc30623dac051e8928ae498cc309967115fd0cb2f0be4d7cf01377e5e449523a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.