| """Read-only, bounded inventory and documented geometry semantics of local sources. |
| |
| Inventory is not a full numerical/mesh audit of every large dataset. CSV counts |
| are measured; supplied manifests and README totals retain their evidence label. |
| Only MultiScan's admitted annotations undergo full per-row geometry validation. |
| """ |
| from __future__ import annotations |
|
|
| import argparse |
| import csv |
| import hashlib |
| import json |
| from pathlib import Path |
|
|
| TOP_DIRECTORIES = ("BillLin66__3D_Room_Collections", "imChuling__3D_Room_Collections", |
| "liantian__3D_Room_Scene_Collections", "XXXpilar__hssd_clean", |
| "XXXpilar__multiscan-clean") |
| BILL, IM, SAGE = TOP_DIRECTORIES[:3] |
| CANONICAL_SOURCE_ROOT = Path("/Volumes/harddisk/3D_Room_Collections") |
|
|
|
|
| def _source(name, directory, sample_file, **fields): |
| return {"name": name, "directory": directory, "sample_file": sample_file, |
| "category_description": "source category; no trustworthy scene-language assumed", |
| "object_asset_id": "instance ID retained in provenance only; asset IDs never enter main condition", |
| "support": "unknown unless explicit source metadata", "relations": "not used as condition by default", |
| "semantic_front": "not independently established for all assets", |
| "eligibility": "audit_only", **fields} |
|
|
|
|
| def source_catalog(): |
| """Versioned semantics, established from local raw records and export code.""" |
| return [ |
| _source("3D-FRONT (BillLin66)", f"{BILL}/3DFront_exported", "3dfront.jsonl", |
| room_geometry="floor mesh convex hull; planar floor/ceiling height; openings", |
| room_type="source room.type", size_semantics="asset-definition local size/bbox times instance scale", |
| extent_kind="full; source bbox variants require independent verification", units="meters", |
| position_reference="asset pivot/origin, not proven bottom/center", rotation="raw quaternion retained in houses; export yaw degrees", |
| up_handedness="source Y-up -> (x,-z,y), right-handed Z-up", scale="export already applies abs(instance scale)", |
| meshes="room structure vertices retained; furniture mesh not included", |
| blockers="pivot-to-bbox offset and semantic front missing; many category/size nulls", |
| declared_counts={"rooms": 56129, "objects": 533837}, |
| code_evidence=["kits/convert_3dfront.py"]), |
| _source("SpatialGen", f"{BILL}/SpatialGen_exported", "spatialgen.jsonl", |
| room_geometry="floor line loop; known self-intersection scene_00019", room_type="caption-derived mapping", |
| size_semantics="instance-scaled local XYZ full extents", extent_kind="full", units="bbox meters; floor/ceiling millimeters", |
| position_reference="box geometric center", rotation="full transform, mirrored/tilted rows retained", |
| up_handedness="right-handed Z-up; reflection separately recorded", scale="norm of transform axis columns, once", |
| meshes="not present", blockers="official test-only; semantic front must be established; reject tilt/mirrors and invalid rooms", |
| declared_counts={"scenes": 48, "objects": 977}, code_evidence=["kits/convert_spatialgen.py"]), |
| _source("3RScan", f"{IM}/3RScan_exported", "3rscan.json", |
| room_geometry="no true room polygon/height; scan container", room_type="missing", |
| size_semantics="obb.axesLengths along raw normalizedAxes", extent_kind="full", units="source metric OBBs, no export normalization", |
| position_reference="obb.centroid", rotation="raw normalizedAxes; exported yaw null", |
| up_handedness="scan frame, source axes need per-record verification", scale="OBB already observed metric geometry", |
| support="3DSSG attributes/affordances", relations="3DSSG relationships preserved", |
| meshes="not present", blockers="room reconstruction and semantic-front evidence required", |
| declared_counts={"scans": 1482, "objects": 43450}, code_evidence=["kits/extract_3rscan.py"]), |
| _source("ARKitScenes", f"{IM}/ARKitScenes_exported", "arkitscenes.json", |
| room_geometry="no true room boundary/height", room_type="missing", |
| size_semantics="segments.obbAligned.axesLengths local OBB", extent_kind="full", units="meters for obbAligned; raw obb is different", |
| position_reference="export world AABB bottom; raw obbAligned.centroid is OBB center", |
| rotation="normalizedAxes.T; raw orientation includes tilt", up_handedness="right-handed Z-up aligned frame", |
| scale="aligned OBB metric extents; no asset scaling", meshes="not present; RGB keyframes archive only", |
| object_asset_id="modelId custom.abandon is a placeholder, never a real CAD identity", |
| blockers="front/room missing; official visit_id has one cross-split location group", |
| declared_counts={"scans": 5047, "objects": 56605}, code_evidence=["extract_arkitscenes_json.py"]), |
| _source("IL3D (3D-FRONT/HSSD/synthetic)", f"{IM}/IL3D_exported", "3dfront.json", |
| room_geometry="source floor mesh; partial 3D-FRONT height/opening supplement", room_type="source category", |
| size_semantics="layout bbox XYZ mapped to local X/Z/Y; verify asset metadata/model orientation", |
| extent_kind="full", units="meters; USDA output separately multiplies by 100", |
| position_reference="claimed bottom center; requires per-asset pivot verification", |
| rotation="USD rotateXYZ matrix -> Z-up; complete matrix kept", up_handedness="right-handed Y-up -> (x,-z,y)", |
| scale="source bbox vs scale requires asset audit; do not reapply automatically", |
| support="placement_flags available; not mesh support surfaces", meshes="CAD USDZ asset archives not downloaded", |
| blockers="semantic-front/pivot/asset axis audit; source NaNs and extreme outlier; opening world-span defect in v1", |
| declared_counts={"scenes": 27816, "objects": 226978}, code_evidence=["aggregate_scenes.py"]), |
| _source("InteriorGS", f"{IM}/InteriorGS_exported", "interiorgs.json", |
| room_geometry="structure room profiles and holes", room_type="missing", |
| size_semantics="long/short horizontal dimensions recovered from source 8 bbox corners", |
| extent_kind="full", units="meters", position_reference="bbox bottom center", |
| rotation="geometric long-axis angle modulo pi, no semantic front", up_handedness="right-handed Z-up; X right/Y back", |
| scale="observed corners already metric", meshes="not present", |
| blockers="geometry-only yaw cannot become semantic-front label; zero extents and ambiguous room assignments", |
| declared_counts={"scenes": 1000, "rooms": 6607, "objects": 506061}, code_evidence=["aggregate_scenes.py"]), |
| _source("InternScenes (Real2Sim/Gen)", f"{IM}/InternScenes_exported", "internscenes_gen_exported.json", |
| room_geometry="floor convex hull; Real2Sim scan/region containers, Gen region labels", room_type="partial", |
| size_semantics="source bbox[3:6] local extents before full ZXY rotation", extent_kind="full", units="meters", |
| position_reference="bbox geometric center", rotation="bbox angle_z/x/y radians; ZXY order", |
| up_handedness="right-handed Z-up layout; floor mesh Y-up converted", scale="box sizes already instance geometry", |
| meshes="floor geometry only; full asset library absent", blockers="semantic front unverified; v1 geometric fallback must not become a front label", |
| declared_counts={"scenes": 23672, "objects": 798954}, code_evidence=["kits/extract_internscenes_json.py"]), |
| _source("MansionWorld", f"{IM}/MansionWorld_exported", "mansionworld.jsonl", |
| room_geometry="room floor polygon, height, doors/windows from THOR source", room_type="source roomType", |
| size_semantics="Objathor annotation bounding box; floor/wall/surface paths differ", |
| extent_kind="full", units="meters", position_reference="floor/wall bottom center; surface path must inspect raw parent geometry", |
| rotation="source THOR yaw mapped to output degrees", up_handedness="THOR Y-up -> (x,-z,y), right-handed Z-up", |
| scale="metadata bbox; any asset scale must be checked against raw annotation", |
| support="surface_groups with source parent associations", meshes="metadata/annotations only, no glTF/SDF", |
| blockers="per-asset canonical front/pivot and support surfaces require audit; room/asset descriptions may leak target geometry", |
| declared_counts={"buildings": 1000, "rooms": 27884, "objects": 405253}, code_evidence=["kits/convert_mansionworld.py"]), |
| _source("OptiScene / 3D-SynthPlace", f"{IM}/OptiScene_exported", "holodeck.json", |
| room_geometry="source rectangular proxy, no original floor polygon", room_type="source labels", |
| size_semantics="bbox [height,width,depth] claimed local geometry; mesh verification missing", |
| extent_kind="full", units="bbox centimeters; positions meters", position_reference="source_position_anchor, pivot semantics not established", |
| rotation="source rotation.y is horizontal angle; full Euler order unspecified", |
| up_handedness="source Y-up; export (x,z,y) reverses handedness; needs explicit yaw correction", |
| scale="bbox source geometry; no unique future mesh implied", meshes="JSON-only export; scenes.tar.gz and prompts retained", |
| blockers="handedness/yaw/pivot and known invalid geometry; do not reuse prompt bbox as v2 condition", |
| declared_counts={"scenes": 17528, "objects": 147222}, code_evidence=["extract_optiscene_json.py", "OptiScene/gprompt.py"]), |
| _source("SceneCAD / Scan2CAD", f"{IM}/SceneCAD_Scan2CAD_exported", "scenecad_scan2cad_combined.json", |
| room_geometry="SceneCAD shell; scan may have multiple rooms; no true room partition", room_type="ScanNet sceneType", |
| size_semantics="ScanNet objects world AABB; separate CAD local bbox half-extents × TRS scale", |
| extent_kind="AABB full; raw CAD bbox half", units="meters", position_reference="ScanNet AABB bottom; CAD geometric OBB center", |
| rotation="ScanNet yaw unavailable; CAD full transform incl axisAlignment", up_handedness="ShapeNet Y-up -> right-handed Z-up", |
| scale="CAD 2*bbox*scale; no rotated AABB used as local size", meshes="not present", |
| blockers="v1 tilted-CAD defect; semantic front and room grouping need raw-source adapter", |
| declared_counts={"scenes": 1509, "cad_scenes": 1506, "cad_objects": 14225}, code_evidence=["aggregate_scenecad_scan2cad.py"]), |
| _source("SceneSmith", f"{IM}/SceneSmith_exported", "json_single.json", |
| room_geometry="floor-plan mesh AABB; Room/House frames", room_type="frame-name derived", |
| size_semantics="visual glTF/SDF mesh local extent; origin bounds must be retained", |
| extent_kind="full", units="meters", position_reference="DMD asset origin; mixed bottom and centered wall assets", |
| rotation="AngleAxis -> Rz Ry Rx Euler degrees", up_handedness="DMD Z-up; glTF Y-up -> (x,-z,y)", |
| scale="SDF visual scale/node transforms applied during extent extraction", meshes="geometry metadata/DMD; full tar meshes not present", |
| blockers="v1 46 wall assets have wrong center assumption; per-asset bounds/front missing", |
| declared_counts={"scenes": 210, "rooms": 364, "objects": 19360}, code_evidence=["kits/scenesmith_aggregate.py", "kits/scenesmith_extract_geometry_metadata.py"]), |
| _source("SpatialLM", f"{IM}/SpatialLM_exported", "spatiallm.json", |
| room_geometry="wall-loop polygon, known invalid rings; source floor plan", room_type="source room category", |
| size_semantics="Bbox scale_x/y/z local extents", extent_kind="full", units="meters", |
| position_reference="box geometric center", rotation="angle_z radians, yaw-only format", |
| up_handedness="Z-up source frame", scale="box extents already final geometry", meshes="not present; layout archive retained", |
| blockers="semantic front calibrated in v1 but not per-asset proven; cross-export source-house duplicates require common split", |
| declared_counts={"scenes": 12328, "rooms": 54775}, code_evidence=["aggregate_spatiallm.py"]), |
| _source("Structured3D", f"{IM}/Structured3D_exported", "structured3d.jsonl", |
| room_geometry="annotation junction/line floor polygons, opening planes, house-room split", room_type="source semantic room labels", |
| size_semantics="raw_coeffs along raw_basis, not heuristic exported axis reorder", extent_kind="raw half extents", |
| units="raw bbox millimeters; exported meters", position_reference="raw bbox centroid; exported room-local center", |
| rotation="raw basis rows; geometric vs semantic front not guaranteed", up_handedness="right-handed Z-up", |
| scale="raw half extents × 2/1000 once", meshes="not present; annotation and rendered labels retained", |
| blockers="heuristic yaw axes/front, 68 zero-sized boxes, source errata, unknown category/tilt", |
| declared_counts={"houses": 3488, "rooms": 22894, "assigned_objects": 398485}, code_evidence=["kits/convert_structured3d.py"]), |
| _source("SAGE10k", f"{SAGE}/SAGE10k", "json_single.json", |
| room_geometry="authored rectangle/wall loop and ceiling height", room_type="source room_type", |
| size_semantics="local dimensions width/length/height; raw mesh scale/frame must verify", |
| extent_kind="full", units="positions/dimensions meters; placement_constraints XY centimeters", |
| position_reference="asset bottom-center origin; raw source physics pose rotates that origin", |
| rotation="Rz Ry Rx degrees incl real pitch/roll; README nonsemantic-tilt claim is not sufficient", |
| up_handedness="Z-up; right-handed from source matrix/render code", scale="object-generation scale/mesh size requires raw asset check", |
| support="place_id floor/wall/instance retained in per-scene source", relations="placement_constraints include exact target pose: exclude from main condition", |
| meshes="local scene metadata; mesh catalog/semantic-front completeness unverified", |
| blockers="semantic front currently empirical +Y; significant physics tilt must explicitly reject; support surfaces not bbox tops", |
| declared_counts={"scenes": 10000, "rooms": 10027, "objects": 853428}, code_evidence=["aggregate_scenes.py", "sage10k/kits/tex_utils_local.py"]), |
| _source("HSSD", "XXXpilar__hssd_clean", "objects.csv", |
| room_geometry="regions.csv polygon/floor/ceiling; 2351 regions", room_type="human region label", |
| size_semantics="rigid mesh-axis OBB half extents; articulated rows lack box", extent_kind="half", units="meters", |
| position_reference="obb_center; translation is asset_local pivot; openings metadata center may be wrong", |
| rotation="obb_rotation_wxyz; full rigid transform", up_handedness="right-handed Y-up -> (x,-z,y)", |
| scale="obb_half_extents already scaled; NEVER multiply non_uniform_scale again", |
| support="asset support/affordance metadata, not verified mesh surfaces", relations="unavailable", |
| meshes="GLB stages/objects/openings plus URDFs present", blockers="per-asset front/up not retained in catalog; empirical v1 fronts differ; opening/pivot and articulated twins need reliable repair", |
| declared_counts={"scenes": 168, "regions": 2351, "objects": 56647}, code_evidence=[]), |
| _source("MultiScan", "XXXpilar__multiscan-clean", "objects.csv", |
| room_geometry="partial floor-slab convex hull; slab top not a verified contact plane", room_type="source scan room_type", |
| size_semantics="annotation OBB axes local extents plus annotated front/up", extent_kind="half", units="meters", |
| position_reference="obb_center -> each own full-height bottom center", rotation="obb_axes plus annotated front/up", |
| up_handedness="right-handed aligned Z-up", scale="annotated box already metric; no extra scale", |
| semantic_front="explicit source front vector; admitted only horizontal OBB-aligned fronts", |
| support="geometry-derived rests_on_floor/supported_by; not trusted physical labels", |
| relations="geometry-derived; omitted to prevent reference-layout leakage", meshes="273 PLY+textured OBJ scans, extracted per-object assets not built", |
| blockers="strict tilt/frame reject; incomplete boundary/floor prevent full room-validity claims", |
| eligibility="full_geometry_supervision_after_strict_filter", declared_counts={"scans": 273, "scenes": 117, "objects": 10957}, |
| code_evidence=[]), |
| ] |
|
|
|
|
| def _file_evidence(path): |
| text = path.read_bytes() |
| return {"path": path.name, "bytes": len(text), "sha256": hashlib.sha256(text).hexdigest()} |
|
|
|
|
| def _sample(path): |
| """Read at most 4 MiB of a large array; never load whole corpus into RAM.""" |
| if path.suffix == ".csv": |
| with path.open(newline="") as fh: |
| reader = csv.DictReader(fh) |
| first = next(reader, {}) |
| count = bool(first) + sum(1 for _ in reader) |
| return {"measured_records": count, "first_record_keys": sorted(first)} |
| with path.open(encoding="utf-8") as fh: |
| prefix = fh.read(4 * 1024 * 1024).lstrip() |
| prefix = prefix[1:].lstrip() if prefix.startswith("[") else prefix |
| first, _ = json.JSONDecoder().raw_decode(prefix) |
| rooms = first.get("rooms") or ([first["room"]] if "room" in first else []) |
| room = rooms[0] if rooms else first.get("room_context", {}) |
| objects = room.get("furniture") or first.get("layout", {}).get("floor_layout", {}).get("objects") or [] |
| return {"first_record_keys": sorted(first), "first_room_keys": sorted(room), |
| "first_object_keys": sorted(objects[0]) if objects else [], "sampling": "first record only; max 4 MiB"} |
|
|
|
|
| def audit_sources(source_root): |
| root = Path(source_root).resolve() |
| entries = [] |
| for definition in source_catalog(): |
| base = root / definition["directory"] |
| evidence = [] |
| for rel in ["README.md", "manifest.json", "dataset_manifest.json", *definition["code_evidence"]]: |
| path = base / rel |
| if path.is_file(): |
| evidence.append({**_file_evidence(path), "path": rel}) |
| sample = {} |
| path = base / definition["sample_file"] |
| if path.is_file(): |
| try: |
| sample = _sample(path) |
| except (ValueError, TypeError, json.JSONDecodeError) as exc: |
| sample = {"sample_error": str(exc)} |
| entries.append({**definition, "present": base.is_dir(), "evidence": evidence, "actual_sample": sample, |
| "count_scope": "declared_counts are retained source documentation, not a new full-corpus recount"}) |
| return {"schema_version": "fastfill.v2.source_audit", "source_root": str(root), |
| "auditor_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), |
| "top_directories": [{"name": name, "present": (root / name).is_dir()} for name in TOP_DIRECTORIES], |
| "sources": entries, "scope": "README+converter+bounded real record inventory; CSV rows counted; no in-place normalization", |
| "source_data_modified": False} |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument("--source-root", type=Path, default=Path("/Volumes/harddisk/3D_Room_Collections")) |
| parser.add_argument("--output", type=Path, help="New JSON output file outside source data") |
| args = parser.parse_args() |
| report = audit_sources(args.source_root) |
| if args.output: |
| path = args.output.resolve() |
| protected = (args.source_root.resolve(), CANONICAL_SOURCE_ROOT.resolve()) |
| if any(path == root or root in path.parents or path in root.parents for root in protected): |
| raise ValueError("audit output must be outside read-only source data") |
| path.parent.mkdir(parents=True, exist_ok=True) |
| with path.open("x") as fh: |
| json.dump(report, fh, indent=2, ensure_ascii=False, allow_nan=False) |
| fh.write("\n") |
| print(json.dumps({"output": str(path), "sources_present": sum(s["present"] for s in report["sources"])})) |
| else: |
| print(json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
