third_party/sam-3d-objects/notebook/demo_multi_object.ipynb

{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Copyright (c) Meta Platforms, Inc. and affiliates."
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 1. Imports and Model Loading"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "import uuid\n",
                "import imageio\n",
                "import numpy as np\n",
                "from IPython.display import Image as ImageDisplay\n",
                "\n",
                "from inference import Inference, ready_gaussian_for_video_rendering, load_image, load_masks, display_image, make_scene, render_video, interactive_visualizer"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "PATH = os.getcwd()\n",
                "TAG = \"hf\"\n",
                "config_path = f\"{PATH}/../checkpoints/{TAG}/pipeline.yaml\"\n",
                "inference = Inference(config_path, compile=False)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 2. Load input image to lift to 3D (multiple objects)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "IMAGE_PATH = f\"{PATH}/images/shutterstock_stylish_kidsroom_1640806567/image.png\"\n",
                "IMAGE_NAME = os.path.basename(os.path.dirname(IMAGE_PATH))\n",
                "\n",
                "image = load_image(IMAGE_PATH)\n",
                "masks = load_masks(os.path.dirname(IMAGE_PATH), extension=\".png\")\n",
                "display_image(image, masks)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 3. Generate Gaussian Splats"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "outputs = [inference(image, mask, seed=42) for mask in masks]"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 4. Visualize Gaussian Splat of the Scene\n",
                "### a. Animated Gif"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "scene_gs = make_scene(*outputs)\n",
                "# export posed gaussian splatting (as point cloud)\n",
                "scene_gs.save_ply(f\"{PATH}/gaussians/{IMAGE_NAME}_posed.ply\")\n",
                "\n",
                "scene_gs = ready_gaussian_for_video_rendering(scene_gs)\n",
                "# export gaussian splatting (as point cloud)\n",
                "scene_gs.save_ply(f\"{PATH}/gaussians/multi/{IMAGE_NAME}.ply\")\n",
                "\n",
                "video = render_video(\n",
                "    scene_gs,\n",
                "    r=1,\n",
                "    fov=60,\n",
                "    resolution=512,\n",
                ")[\"color\"]\n",
                "\n",
                "# save video as gif\n",
                "imageio.mimsave(\n",
                "    os.path.join(f\"{PATH}/gaussians/multi/{IMAGE_NAME}.gif\"),\n",
                "    video,\n",
                "    format=\"GIF\",\n",
                "    duration=1000 / 30,  # default assuming 30fps from the input MP4\n",
                "    loop=0,  # 0 means loop indefinitely\n",
                ")\n",
                "\n",
                "# notebook display\n",
                "ImageDisplay(url=f\"gaussians/multi/{IMAGE_NAME}.gif?cache_invalidator={uuid.uuid4()}\",)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### b. Interactive Visualizer"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# might take a while to load (black screen)\n",
                "interactive_visualizer(f\"{PATH}/gaussians/multi/{IMAGE_NAME}.ply\")"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.11.0"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}