Buckets:
| #!/usr/bin/env python3 | |
| """bucket_audit — 盘点 /domains 的规模与构成，判断能否作为数据集仓库发布。""" | |
| from collections import defaultdict | |
| from pathlib import Path | |
| ROOT = Path("/domains") | |
| top_stats = defaultdict(lambda: [0, 0]) # name -> [files, bytes] | |
| ext_stats = defaultdict(lambda: [0, 0]) | |
| pdfs = [] | |
| big = [] | |
| total_files = 0 | |
| total_bytes = 0 | |
| for p in ROOT.rglob("*"): | |
| if not p.is_file(): | |
| continue | |
| try: | |
| sz = p.stat().st_size | |
| except OSError: | |
| continue | |
| rel = p.relative_to(ROOT) | |
| top = rel.parts[0] + ("/" if len(rel.parts) > 1 else "") | |
| top_stats[top][0] += 1 | |
| top_stats[top][1] += sz | |
| ext = p.suffix.lower() or "(none)" | |
| ext_stats[ext][0] += 1 | |
| ext_stats[ext][1] += sz | |
| total_files += 1 | |
| total_bytes += sz | |
| if ext == ".pdf": | |
| pdfs.append(str(rel)) | |
| if sz > 200 * 1024 * 1024: | |
| big.append((str(rel), sz)) | |
| print(f"TOTAL files={total_files} bytes={total_bytes} ({total_bytes/2**30:.2f} GiB)") | |
| print("\n--- 顶层目录 / 文件（按字节降序）---") | |
| for k, (n, b) in sorted(top_stats.items(), key=lambda kv: -kv[1][1]): | |
| print(f"{b/2**20:10.1f} MiB {n:7d} {k}") | |
| print("\n--- 扩展名 ---") | |
| for k, (n, b) in sorted(ext_stats.items(), key=lambda kv: -kv[1][1])[:15]: | |
| print(f"{b/2**20:10.1f} MiB {n:7d} {k}") | |
| print(f"\n--- PDF 文件（应只有论文类，不应进公开仓库）: {len(pdfs)} ---") | |
| for x in pdfs[:20]: | |
| print(" " + x) | |
| print(f"\n--- >200 MiB 的单文件: {len(big)} ---") | |
| for name, sz in big[:20]: | |
| print(f" {sz/2**30:6.2f} GiB {name}") | |
Xet Storage Details
- Size:
- 1.59 kB
- Xet hash:
- 93b1ad12a7eb9c211e203cb94aab3b95dc765b42038f1f625cb61bc3c3cd0b7b
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.