Buckets:
| """Propose class-supported development groups while quarantining old audit groups.""" | |
| import argparse | |
| from concurrent.futures import ThreadPoolExecutor | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| from xml.etree import ElementTree | |
| AIRCRAFT = {"Boeing737", "Boeing747", "Boeing777", "Boeing787", "ARJ21", "C919", | |
| "A220", "A321", "A330", "A350", "other-airplane"} | |
| def old_split(stem): | |
| value = int(hashlib.sha256(("fair1m-v1:" + stem).encode()).hexdigest()[:16], 16) % 10 | |
| return "audit" if value == 0 else "val" if value == 1 else "train" | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("groups", type=Path) | |
| parser.add_argument("labels", type=Path) | |
| parser.add_argument("output", type=Path) | |
| args = parser.parse_args() | |
| groups = json.loads(args.groups.read_text())["groups"] | |
| partitions = {"train": [], "val": [], "audit": []} | |
| for group in groups: | |
| former = {old_split(stem) for stem in group} | |
| key = "audit" if "audit" in former else "val" if "val" in former else "train" | |
| partitions[key].append(group) | |
| def read(stem): | |
| root = ElementTree.parse(args.labels / f"{stem}.xml").getroot() | |
| return stem, {node.text for node in root.findall("./objects/object/possibleresult/name")} | |
| development = [stem for key in ("train", "val") for group in partitions[key] for stem in group] | |
| with ThreadPoolExecutor(max_workers=8) as executor: | |
| names = dict(executor.map(read, development)) | |
| moved = [] | |
| for label, members, minimum in (("aircraft", AIRCRAFT, 2), ("bridge", {"Bridge"}, 10)): | |
| count = lambda pool: sum(bool(names[stem] & members) for group in pool for stem in group) | |
| for group in sorted(partitions["train"], key=lambda g: hashlib.sha256((label + ":" + g[0]).encode()).hexdigest()): | |
| if count(partitions["val"]) >= minimum: | |
| break | |
| if any(names[stem] & members for stem in group): | |
| partitions["train"].remove(group) | |
| partitions["val"].append(group) | |
| moved.append({"reason": label, "ids": group}) | |
| result = {"status": "PROPOSED_ONLY", "group_registry_sha256": hashlib.sha256(args.groups.read_bytes()).hexdigest(), | |
| "groups": partitions, "moved_to_validation": moved, | |
| "validation_source_support": {"aircraft": sum(bool(names[s] & AIRCRAFT) for g in partitions["val"] for s in g), | |
| "bridge": sum("Bridge" in names[s] for g in partitions["val"] for s in g)}, | |
| "limitations": ["Development proposal only; no independent test claim.", | |
| "No nearby-scene, source-overlap, pretraining-exposure, or license verification."]} | |
| with args.output.open("x") as stream: | |
| json.dump(result, stream, indent=2) | |
| stream.write("\n") | |
| print(json.dumps({"counts": {key: sum(map(len, value)) for key, value in partitions.items()}, | |
| "validation_source_support": result["validation_source_support"], "moved_groups": len(moved)})) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 3.13 kB
- Xet hash:
- da14645526c45013475bde3b0ad012e6ab22dde99005691f1b308f8755bde891
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.