third_party/sam-3d-objects/notebook/demo_single_object.ipynb

{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Copyright (c) Meta Platforms, Inc. and affiliates."
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 1. Imports and Model Loading"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "import imageio\n",
                "import uuid\n",
                "from IPython.display import Image as ImageDisplay\n",
                "from inference import Inference, ready_gaussian_for_video_rendering, render_video, load_image, load_single_mask, display_image, make_scene, interactive_visualizer"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "PATH = os.getcwd()\n",
                "TAG = \"hf\"\n",
                "config_path = f\"{PATH}/../checkpoints/{TAG}/pipeline.yaml\"\n",
                "inference = Inference(config_path, compile=False)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 2. Load input image to lift to 3D (single object)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "IMAGE_PATH = f\"{PATH}/images/shutterstock_stylish_kidsroom_1640806567/image.png\"\n",
                "IMAGE_NAME = os.path.basename(os.path.dirname(IMAGE_PATH))\n",
                "\n",
                "image = load_image(IMAGE_PATH)\n",
                "mask = load_single_mask(os.path.dirname(IMAGE_PATH), index=14)\n",
                "display_image(image, masks=[mask])"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 3. Generate Gaussian Splat"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# run model\n",
                "output = inference(image, mask, seed=42)\n",
                "\n",
                "# export gaussian splat (as point cloud)\n",
                "output[\"gs\"].save_ply(f\"{PATH}/gaussians/single/{IMAGE_NAME}.ply\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## 4. Visualize Gaussian Splat\n",
                "### a. Animated Gif"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# render gaussian splat\n",
                "scene_gs = make_scene(output)\n",
                "scene_gs = ready_gaussian_for_video_rendering(scene_gs)\n",
                "\n",
                "video = render_video(\n",
                "    scene_gs,\n",
                "    r=1,\n",
                "    fov=60,\n",
                "    pitch_deg=15,\n",
                "    yaw_start_deg=-45,\n",
                "    resolution=512,\n",
                ")[\"color\"]\n",
                "\n",
                "# save video as gif\n",
                "imageio.mimsave(\n",
                "    os.path.join(f\"{PATH}/gaussians/single/{IMAGE_NAME}.gif\"),\n",
                "    video,\n",
                "    format=\"GIF\",\n",
                "    duration=1000 / 30,  # default assuming 30fps from the input MP4\n",
                "    loop=0,  # 0 means loop indefinitely\n",
                ")\n",
                "\n",
                "# notebook display\n",
                "ImageDisplay(url=f\"gaussians/single/{IMAGE_NAME}.gif?cache_invalidator={uuid.uuid4()}\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### b. Interactive Visualizer"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# might take a while to load (black screen)\n",
                "interactive_visualizer(f\"{PATH}/gaussians/single/{IMAGE_NAME}.ply\")"
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "sam3d_objects-3dfy",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.11.0"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}