| """ |
| Launches a diagnostic FiftyOne App for a Hugging Face Docker Space. |
| |
| The application downloads Hub datasets into ephemeral local storage, imports |
| their records into MongoDB, and creates browser-local clones on demand. |
| """ |
|
|
| import asyncio |
| import faulthandler |
| import importlib.metadata |
| import json |
| import logging |
| import os |
| import platform |
| import shutil |
| import socket |
| import sys |
| import time |
|
|
| from hypercorn.asyncio import serve |
| from hypercorn.config import Config |
| from huggingface_hub import snapshot_download |
|
|
| import fiftyone as fo |
|
|
| from gateway import create_gateway |
|
|
|
|
| EXTERNAL_APP_ADDRESS = "0.0.0.0" |
| EXTERNAL_APP_PORT = int(os.environ.get("PORT", "7860")) |
| INTERNAL_APP_ADDRESS = "127.0.0.1" |
| INTERNAL_APP_PORT = 5151 |
| MANIFEST_PATH = os.path.join(os.path.dirname(__file__), "datasets.json") |
| LOG_FORMAT = "%(asctime)s | %(levelname)s | %(name)s | %(message)s" |
|
|
| logging.basicConfig( |
| level=os.environ.get("LOG_LEVEL", "INFO").upper(), |
| format=LOG_FORMAT, |
| stream=sys.stdout, |
| force=True, |
| ) |
| logger = logging.getLogger(__name__) |
|
|
|
|
| def main(): |
| """Loads the configured datasets and serves the FiftyOne App.""" |
| faulthandler.enable(file=sys.stderr, all_threads=True) |
| faulthandler.dump_traceback_later(300, repeat=True, file=sys.stderr) |
|
|
| _log_runtime_diagnostics() |
|
|
| try: |
| manifest = _load_manifest() |
| datasets = _load_datasets(manifest) |
| session = _launch_app(datasets[manifest["default_dataset"]]) |
| except Exception: |
| logger.exception("Fatal error while starting the FiftyOne App") |
| raise |
| finally: |
| faulthandler.cancel_dump_traceback_later() |
|
|
| logger.info( |
| "FiftyOne App is ready; starting the browser session gateway" |
| ) |
| try: |
| asyncio.run(_serve_gateway(datasets, manifest)) |
| finally: |
| session.close() |
|
|
|
|
| def _load_manifest(): |
| """Loads and validates the dataset manifest.""" |
| with open(MANIFEST_PATH) as file: |
| manifest = json.load(file) |
|
|
| entries = manifest.get("datasets") |
| if not isinstance(entries, list) or not entries: |
| raise ValueError("datasets.json must contain at least one dataset") |
|
|
| required_fields = { |
| "name", |
| "repo_id", |
| "local_path", |
| "expected_samples", |
| } |
| names = [] |
| for entry in entries: |
| missing = required_fields - entry.keys() |
| if missing: |
| raise ValueError( |
| f"Dataset manifest entry is missing fields: {sorted(missing)}" |
| ) |
|
|
| if not os.path.isabs(entry["local_path"]): |
| raise ValueError( |
| f"Dataset local_path must be absolute: {entry['local_path']}" |
| ) |
|
|
| if not entry["name"] or "/" in entry["name"]: |
| raise ValueError( |
| f"Dataset name must be a nonempty path segment: {entry['name']}" |
| ) |
|
|
| if ( |
| not isinstance(entry["expected_samples"], int) |
| or entry["expected_samples"] < 0 |
| ): |
| raise ValueError( |
| "Dataset expected_samples must be a nonnegative integer: " |
| f"{entry['expected_samples']}" |
| ) |
|
|
| names.append(entry["name"]) |
|
|
| if len(names) != len(set(names)): |
| raise ValueError("Dataset names in datasets.json must be unique") |
|
|
| if manifest.get("default_dataset") not in names: |
| raise ValueError( |
| "default_dataset must match a dataset name in datasets.json" |
| ) |
|
|
| logger.info( |
| "Dataset manifest loaded: default=%s datasets=%s", |
| manifest["default_dataset"], |
| names, |
| ) |
| return manifest |
|
|
|
|
| def _load_datasets(manifest): |
| """Downloads and imports fresh datasets from the Hugging Face Hub.""" |
| logger.info("Connecting to the FiftyOne database") |
| existing_datasets = fo.list_datasets() |
| logger.info( |
| "Database connection succeeded; existing datasets=%s", |
| existing_datasets, |
| ) |
|
|
| entries = manifest["datasets"] |
| session_prefixes = tuple( |
| entry["name"] + "-session-" for entry in entries |
| ) |
| stale_clones = [ |
| name |
| for name in existing_datasets |
| if name.startswith(session_prefixes) |
| ] |
| for name in stale_clones: |
| logger.info("Deleting stale browser dataset clone '%s'", name) |
| fo.delete_dataset(name) |
|
|
| for entry in entries: |
| dataset_name = entry["name"] |
| if dataset_name in existing_datasets: |
| logger.info("Deleting stale base dataset '%s'", dataset_name) |
| fo.delete_dataset(dataset_name) |
|
|
| datasets = {} |
| for entry in entries: |
| dataset_name = entry["name"] |
| dataset_dir = entry["local_path"] |
| expected_samples = entry["expected_samples"] |
| logger.info( |
| "Downloading Hub dataset: repo=%s directory=%s", |
| entry["repo_id"], |
| dataset_dir, |
| ) |
| download_started_at = time.monotonic() |
| snapshot_download( |
| repo_id=entry["repo_id"], |
| repo_type="dataset", |
| local_dir=dataset_dir, |
| ) |
| logger.info( |
| "Hub dataset downloaded in %.2f seconds: repo=%s directory=%s", |
| time.monotonic() - download_started_at, |
| entry["repo_id"], |
| dataset_dir, |
| ) |
|
|
| logger.info( |
| "Importing local dataset: name=%s directory=%s", |
| dataset_name, |
| dataset_dir, |
| ) |
| started_at = time.monotonic() |
| dataset = fo.Dataset.from_dir( |
| dataset_dir=dataset_dir, |
| dataset_type=fo.types.FiftyOneDataset, |
| name=dataset_name, |
| persistent=True, |
| ) |
| elapsed = time.monotonic() - started_at |
|
|
| logger.info( |
| "Dataset imported in %.2f seconds: name=%s samples=%d " |
| "media_type=%s", |
| elapsed, |
| dataset.name, |
| len(dataset), |
| dataset.media_type, |
| ) |
| if len(dataset) != expected_samples: |
| raise RuntimeError( |
| f"{dataset_name} import count mismatch: expected " |
| f"{expected_samples}, found {len(dataset)}" |
| ) |
|
|
| logger.info( |
| "Dataset validation succeeded: name=%s expected_episodes=%d", |
| dataset_name, |
| expected_samples, |
| ) |
| logger.info( |
| "Dataset schema: name=%s schema=%s", |
| dataset_name, |
| dataset.get_field_schema(), |
| ) |
| logger.info( |
| "Dataset disk usage: name=%s stats=%s", |
| dataset_name, |
| dataset.stats(include_media=True), |
| ) |
|
|
| first_sample = dataset.first() |
| logger.info( |
| "First sample diagnostics: dataset=%s id=%s filepath=%s " |
| "file_exists=%s", |
| dataset_name, |
| first_sample.id, |
| first_sample.filepath, |
| os.path.isfile(first_sample.filepath), |
| ) |
| datasets[dataset_name] = dataset |
|
|
| return datasets |
|
|
|
|
| def _launch_app(dataset): |
| """Launches the internal App behind the browser session gateway.""" |
| logger.info( |
| "Launching internal FiftyOne App: address=%s port=%d remote=True", |
| INTERNAL_APP_ADDRESS, |
| INTERNAL_APP_PORT, |
| ) |
| started_at = time.monotonic() |
| session = fo.launch_app( |
| dataset, |
| address=INTERNAL_APP_ADDRESS, |
| port=INTERNAL_APP_PORT, |
| remote=True, |
| ) |
| elapsed = time.monotonic() - started_at |
| logger.info( |
| "FiftyOne session launched in %.2f seconds: url=%s server=%s:%s", |
| elapsed, |
| session.url, |
| session.server_address, |
| session.server_port, |
| ) |
| return session |
|
|
|
|
| async def _serve_gateway(datasets, manifest): |
| """Serves the external session-aware gateway.""" |
| config = Config() |
| config.bind = [f"{EXTERNAL_APP_ADDRESS}:{EXTERNAL_APP_PORT}"] |
| config.accesslog = "-" |
| config.errorlog = "-" |
| config.graceful_timeout = 10 |
| config.keep_alive_timeout = 30 |
| logger.info( |
| "Serving session gateway: address=%s port=%d", |
| EXTERNAL_APP_ADDRESS, |
| EXTERNAL_APP_PORT, |
| ) |
| shared_media_roots = [ |
| entry["local_path"] for entry in manifest["datasets"] |
| ] |
| await serve( |
| create_gateway( |
| datasets, |
| manifest["default_dataset"], |
| shared_media_roots, |
| ), |
| config, |
| ) |
|
|
|
|
| def _log_runtime_diagnostics(): |
| """Logs non-secret runtime details useful for diagnosing Space failures.""" |
| disk = shutil.disk_usage("/") |
| package_names = ( |
| "fiftyone", |
| "fiftyone-db", |
| "pymongo", |
| "starlette", |
| "hypercorn", |
| "httpx", |
| ) |
| package_versions = {} |
| for package_name in package_names: |
| try: |
| package_versions[package_name] = importlib.metadata.version( |
| package_name |
| ) |
| except importlib.metadata.PackageNotFoundError: |
| package_versions[package_name] = "not-installed" |
|
|
| logger.info("Starting FiftyOne Hugging Face Space") |
| logger.info( |
| "Runtime: python=%s executable=%s platform=%s architecture=%s", |
| sys.version.replace("\n", " "), |
| sys.executable, |
| platform.platform(), |
| platform.machine(), |
| ) |
| logger.info( |
| "Process: pid=%d uid=%d gid=%d hostname=%s cwd=%s", |
| os.getpid(), |
| os.getuid(), |
| os.getgid(), |
| socket.gethostname(), |
| os.getcwd(), |
| ) |
| logger.info( |
| "Environment: HOME=%s PORT=%s LOG_LEVEL=%s", |
| os.environ.get("HOME"), |
| os.environ.get("PORT"), |
| os.environ.get("LOG_LEVEL"), |
| ) |
| logger.info("Package versions=%s", package_versions) |
| logger.info( |
| "Disk '/': total=%.2fGiB used=%.2fGiB free=%.2fGiB", |
| disk.total / 1024**3, |
| disk.used / 1024**3, |
| disk.free / 1024**3, |
| ) |
| logger.info( |
| "FiftyOne config: database_dir=%s database_uri=%s " |
| "dataset_zoo_dir=%s model_zoo_dir=%s default_app_port=%s " |
| "default_app_address=%s", |
| getattr(fo.config, "database_dir", None), |
| getattr(fo.config, "database_uri", None), |
| getattr(fo.config, "dataset_zoo_dir", None), |
| getattr(fo.config, "model_zoo_dir", None), |
| getattr(fo.config, "default_app_port", None), |
| getattr(fo.config, "default_app_address", None), |
| ) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
