File size: 6,919 Bytes
e6f80f3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | """Build immutable FastFill-v2 splits from read-only audited annotations."""
from __future__ import annotations
import argparse
import hashlib
import json
from collections import Counter
from contextlib import ExitStack
from pathlib import Path
from fastfill.v2.adapters import iter_multiscan
DEFAULT_SOURCE_ROOT = Path("/Volumes/harddisk/3D_Room_Collections")
def grouped_split(source: str, house_id: str, seed: int = 42) -> str:
"""Stable 80/10/10 partition of source/house identity before derivation."""
key = json.dumps([seed, source, str(house_id)], separators=(",", ":")).encode()
fraction = int.from_bytes(hashlib.sha256(key).digest()[:8], "big") / 2**64
return "train" if fraction < .8 else "validation" if fraction < .9 else "test"
def filter_rows_by_flags(rows, flags):
"""Drop rows whose provenance.legacy_flags report any listed flag (same truthiness rule as the train build)."""
if not isinstance(flags, (list, tuple)) or any(not isinstance(flag, str) or not flag for flag in flags):
raise ValueError("exclude_flags must be a list of nonempty flag names")
return [row for row in rows
if not any(row.get("provenance", {}).get("legacy_flags", {}).get(flag) for flag in flags)]
def _digest(path):
digest = hashlib.sha256()
with Path(path).open("rb") as fh:
for block in iter(lambda: fh.read(1024 * 1024), b""):
digest.update(block)
return digest.hexdigest()
def _output_path(source_root, output):
source, target = Path(source_root).resolve(), Path(output).resolve()
protected = (source, DEFAULT_SOURCE_ROOT.resolve())
if any(target == root or root in target.parents or target in root.parents for root in protected):
raise ValueError("output must be an explicitly new path outside source data and its ancestors")
if target.exists():
raise FileExistsError(f"immutable build output already exists: {target}")
return source, target
def build_dataset(source_root, output, *, sources=("multiscan",), seed=42, max_scenes=None):
"""Build admitted sources without overwrite or writes beneath source_root.
Only raw MultiScan geometry is admitted initially. All other local sources
remain audit-only until their canonical front/pivot/scale blockers are
resolved. `max_scenes` bounds admitted output, not changes the split rule.
"""
source, target = _output_path(source_root, output)
if tuple(sources) != ("multiscan",):
raise ValueError("currently audited builder source is multiscan; uncertain sources are not admitted")
if max_scenes is not None and max_scenes < 1:
raise ValueError("max_scenes must be positive")
# Check readable source files before creating output; sources stay read-only.
for name in ("objects.csv", "regions.csv", "scans.csv"):
if not (source / "XXXpilar__multiscan-clean" / name).is_file():
raise FileNotFoundError(f"required raw source missing: {name}")
target.mkdir(parents=True, exist_ok=False)
counts, objects, houses = Counter(), Counter(), {}
diagnostics = {"scenes_inspected": 0, "rejections": []}
with ExitStack() as stack:
handles = {split: stack.enter_context((target / f"{split}.jsonl").open("x"))
for split in ("train", "validation", "test")}
for sample in iter_multiscan(source, diagnostics):
provenance = sample["provenance"]
split = grouped_split(provenance["source"], provenance["house_id"], seed)
houses[f"{provenance['source']}:{provenance['house_id']}"] = split
record = {**sample, "provenance": {**provenance, "split": split}}
handles[split].write(json.dumps(record, allow_nan=False, separators=(",", ":")) + "\n")
counts[split] += 1
objects[split] += len(record["target"]["objects"])
if max_scenes is not None and sum(counts.values()) >= max_scenes:
break
manifest = {"schema_version": "fastfill.v2", "source_root": str(source), "builder_version": 1,
"sources": list(sources), "samples_written": sum(counts.values()),
"objects_written": sum(objects.values()), "split_samples": dict(counts),
"split_objects": dict(objects), "split_seed": seed, "split_rule":
"SHA256(seed, source, underlying scene/house) -> 80/10/10 before derivation",
"house_split": houses, "bounded_build": max_scenes is not None,
"source_data_modified": False, "diagnostics": diagnostics,
"source_sha256": {name: _digest(source / "XXXpilar__multiscan-clean" / name)
for name in ("objects.csv", "regions.csv", "scans.csv")},
"adapter_sha256": _digest(Path(__file__).with_name("adapters.py")),
"builder_sha256": _digest(__file__),
"boundary_validation_eligible": False, "eligibility":
"upright annotated-front local OBB supervision; scan floor hull/contact remain incomplete"}
with (target / "manifest.json").open("x") as fh:
json.dump(manifest, fh, indent=2, allow_nan=False)
fh.write("\n")
return manifest
def main(argv=None):
from fastfill.v2.legacy_build import DEFAULT_RELEASE, DEFAULT_EVIDENCE
from fastfill.v2.legacy_bridge import FRONT_POLICIES
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-root", type=Path, default=DEFAULT_SOURCE_ROOT)
parser.add_argument("--output", type=Path, required=True, help="New immutable output directory")
parser.add_argument("--source", choices=["selected-v3.2", "multiscan"], default="selected-v3.2")
parser.add_argument("--release-root", type=Path, default=DEFAULT_RELEASE)
parser.add_argument("--evidence-root", type=Path, default=DEFAULT_EVIDENCE)
parser.add_argument("--front-policy", choices=FRONT_POLICIES, default="axis")
parser.add_argument("--include-flagged", action="store_true")
parser.add_argument("--workers", type=int, default=1)
parser.add_argument("--seed", type=int, default=42)
parser.add_argument("--max-scenes", type=int)
args = parser.parse_args(argv)
if args.source == "selected-v3.2":
from fastfill.v2.legacy_build import build_selected_dataset
manifest = build_selected_dataset(args.release_root, args.output, evidence_root=args.evidence_root,
seed=args.seed, max_scenes=args.max_scenes, front_policy=args.front_policy,
include_flagged=args.include_flagged, workers=args.workers)
else:
manifest = build_dataset(args.source_root, args.output, sources=(args.source,),
seed=args.seed, max_scenes=args.max_scenes)
print(json.dumps({key: manifest[key] for key in ("samples_written", "split_samples")}, indent=2))
if __name__ == "__main__":
main()
|