app.py

#!/usr/bin/env python

"""
FingerNet Demo App
"""

import os
import numpy as np
import trimesh
import pandas as pd
import tempfile
import gradio as gr
import matplotlib as mpl
from huggingface_hub import hf_hub_download
import onnxruntime as ort

model_dict = {
    "FingerNet": "fingernet",
    "FingerNet-Surf": "fingernet-surf",
}

vertices_dict = {}
faces_dict = {}
def_index_dict = {}
for name in model_dict.keys():
    vertices_dict[name] = np.loadtxt(
        f"assets/{name.lower().replace('net', '')}/surf_coor.txt", delimiter=","
    )
    faces_dict[name] = (
        np.loadtxt(
            f"assets/{name.lower().replace('net', '')}/surf_tri.txt",
            delimiter=",",
            dtype=int,
        )
        - 1
    )
    def_index_dict[name] = np.loadtxt(
        f"assets/{name.lower().replace('net', '')}/def_node.txt", dtype=int
    )
colormap = mpl.colormaps.get_cmap("viridis")
cmin = 0.0
cmax = 12.0

def load_model(model_name="fingernet"):
    repo_id = f"asRobotics/{model_name}"
    model_filename = "model.onnx"
    model_path = hf_hub_download(repo_id=repo_id, filename=model_filename)
    return ort.InferenceSession(model_path)

ort_dict = {}
for name, path in model_dict.items():
    try:
        ort_dict[name] = load_model(name)
    except Exception as e:
        print(f"Warning: Could not load model '{name}' from '{path}'. Error: {e}")


def make_double_sided(mesh: trimesh.Trimesh) -> trimesh.Trimesh:
    faces_front = mesh.faces.copy()
    vertices = mesh.vertices.copy()
    faces_back = np.fliplr(faces_front) + len(vertices)
    vertices_double = np.vstack([vertices, vertices])
    faces_double = np.vstack([faces_front, faces_back])

    if hasattr(mesh.visual, "vertex_colors") and mesh.visual.vertex_colors is not None:
        colors = mesh.visual.vertex_colors.copy()
        colors_double = np.vstack([colors, colors])
    else:
        colors_double = None

    mesh_double = trimesh.Trimesh(
        vertices=vertices_double,
        faces=faces_double,
        vertex_colors=colors_double,
        process=False,
    )

    return mesh_double


def predict_from_motion(dx, dy, dz, rx, ry, rz, model_name):
    motion = np.array(
        [dx, dy, dz, rx / 180.0 * np.pi, ry / 180.0 * np.pi, rz / 180.0 * np.pi],
        dtype=np.float32,
    ).reshape(1, -1)
    assert motion.shape[1] == 6, "Motion must be 6D (x,y,z,rx,ry,rz)"

    outputs = ort_dict[model_name].run(None, {"motion": motion})
    force = np.array(outputs[0]).flatten()
    nodes = np.array(outputs[1]).reshape(-1, 3)

    df = pd.DataFrame(
        force.reshape(1, -1),
        columns=["Fx (N)", "Fy (N)", "Fz (N)", "Tx (Nmm)", "Ty (Nmm)", "Tz (Nmm)"],
    )
    df = df.style.format("{:.3f}")

    delta_vertices = np.zeros_like(vertices_dict[model_name])
    delta_vertices[def_index_dict[model_name] - 1] += nodes.reshape(-1, 3)
    new_vertices = vertices_dict[model_name] + delta_vertices

    rot_x_neg90 = np.array(
        [
            [1, 0, 0],
            [0, 0, 1],
            [0, -1, 0],
        ]
    )
    new_vertices = new_vertices @ rot_x_neg90.T

    norm_dist = np.clip(np.linalg.norm(delta_vertices, axis=1), cmin, cmax)
    colors = (colormap((norm_dist - cmin) / (cmax - cmin)) * 255).astype(np.uint8)

    mesh = trimesh.Trimesh(
        vertices=new_vertices,
        faces=faces_dict[model_name],
        vertex_colors=colors,
        process=False,
    )
    mesh = make_double_sided(mesh)

    tmp_file = tempfile.NamedTemporaryFile(delete=False, suffix=".glb")
    mesh.export(tmp_file.name, file_type="glb")

    return df, tmp_file.name


with gr.Blocks(title="FingerNet Demo") as demo:
    gr.Markdown("# 🖐️ FingerNet Demo\n\n")

    with gr.Row():
        with gr.Column():
            gr.Markdown("## Model Selection")
            model = gr.Dropdown(
                choices=list(model_dict.keys()),
                value=list(model_dict.keys())[0],
                label="Select Model",
                show_label=False,
            )

            gr.Markdown("## Motion Inputs")
            gr.Markdown("### Translation (mm)")
            dx = gr.Slider(-10, 10, 0, step=0.1, label="Dx")
            dy = gr.Slider(-10, 10, 0, step=0.1, label="Dy")
            dz = gr.Slider(-3, 3, 0, step=0.1, label="Dz")
            gr.Markdown("### Rotation (degrees)")
            rx = gr.Slider(-30, 30, 0, step=0.5, label="Rx")
            ry = gr.Slider(-30, 30, 0, step=0.5, label="Ry")
            rz = gr.Slider(-30, 30, 0, step=0.5, label="Rz")

        with gr.Column():
            gr.Markdown("## Force Outputs")
            output_force = gr.DataFrame(
                headers=[
                    "Fx (N)",
                    "Fy (N)",
                    "Fz (N)",
                    "Tx (Nmm)",
                    "Ty (Nmm)",
                    "Tz (Nmm)",
                ],
                label="Predicted Force",
                show_label=False,
                datatype=["number"] * 6,
                row_count=(1, "fixed"),
                interactive=False,
                column_widths=["16%"] * 6,
            )

            gr.Markdown("## Mesh Outputs")
            height = 500
            output_mesh = gr.Model3D(
                label="Mesh",
                show_label=False,
                height=height,
                camera_position=[45, 60, 150],
            )

    inputs = [dx, dy, dz, rx, ry, rz, model]
    for inp in inputs:
        inp.change(
            fn=predict_from_motion, inputs=inputs, outputs=[output_force, output_mesh]
        )

    demo.load(
        fn=predict_from_motion, inputs=inputs, outputs=[output_force, output_mesh]
    )

demo.launch(server_name="0.0.0.0", server_port=int(os.environ.get("PORT", 7860)))