src/visualize/visualize.py

from typing import Dict, List
import numpy as np
from tqdm import tqdm
import math, json
import textwrap
from matplotlib import pyplot as plt
from matplotlib.animation import FuncAnimation
from matplotlib import gridspec
from matplotlib.font_manager import FontProperties
import matplotlib.gridspec as gridspec
from pygments import highlight
from pygments.lexers import JsonLexer
from pygments.formatter import Formatter
from pygments.styles import get_style_by_name
from pygments.token import Token

# NOTE: bihand_mano2mesh (from .mano2mesh) is imported lazily where used (mano viz mode only),
# so skeleton visualization works without the license-gated MANO model files.
from .mesh_visualizer import Mesh_Visualize_Helper
from .skeleton_visualizer import Skeleton_Visualize_Helper
from ..utils import smart_wrap

class MatplotlibFormatter(Formatter):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.data = []

    def format(self, tokensource, outfile):
        self.data = []
        for ttype, value in tokensource:
            style = self.style.style_for_token(ttype)
            color = style['color']
            if color:
                color = f"#{color}"
            else:

                style_bg_is_dark = self.style.background_color < '#777'
                color = '#ffffff' if style_bg_is_dark else '#000000'

            self.data.append((value, color, ttype))


class JsonPlotter:
    @staticmethod
    def plot_json(
        data_dict: dict | list,
        ax: plt.Axes,
        width: int = 80,
        style_name: str = 'monokai',
        font_family: str = 'monospace',
        font_size: int = 10
    ):

        json_str = json.dumps(data_dict, indent=4, ensure_ascii=False)
        lexer = JsonLexer()
        style = get_style_by_name(style_name)
        formatter = MatplotlibFormatter(style=style)
        highlight(json_str, lexer, formatter)

        ax.set_facecolor(style.background_color)

        fig = ax.get_figure()
        font = FontProperties(family=font_family, size=font_size)


        sample_text_obj = ax.text(0, 0, 'M', fontproperties=font, visible=False)

        try:
            fig.canvas.draw()

            text_bbox = sample_text_obj.get_window_extent()
            ax_bbox = ax.get_window_extent()

            if ax_bbox.width == 0 or text_bbox.width == 0:
                raise ValueError("Canvas has zero width, cannot calculate font metrics.")

            char_width = text_bbox.width / ax_bbox.width
            line_height = text_bbox.height / ax_bbox.height * 1.6

            print(f"Char Width: {char_width}, Line Height: {line_height}")
        except Exception:
            char_width = 0.006 * (font_size / 10)
            line_height = 0.025 * (font_size / 10)
        finally:
            sample_text_obj.remove()


        lines_of_tokens = []
        current_line = []
        current_char_count = 0

        wrapper = textwrap.TextWrapper(
            width=width,
            break_long_words=True,
            break_on_hyphens=False
        )

        for text, color, ttype in formatter.data:
            if text == '\n':
                lines_of_tokens.append(current_line)
                current_line = []
                current_char_count = 0
                continue

            is_string = ttype in Token.Literal.String
            if is_string and (current_char_count + len(text)) > width:
                leading_spaces = current_char_count
                wrapper.initial_indent = " " * 0

                wrapper.subsequent_indent = " " * (leading_spaces + 4)

                content = text[1:-1]
                wrapped_lines = wrapper.wrap(content)

                current_line.append((text[0], color))

                for i, line_content in enumerate(wrapped_lines):
                    if i == 0:

                        current_line.append((line_content, color))
                    else:
                        lines_of_tokens.append(current_line)

                        current_line = [ (line_content.lstrip(), color) ]


                current_line.append((text[-1], color))
            else:
                current_line.append((text, color))
                current_char_count += len(text)

        if current_line:
            lines_of_tokens.append(current_line)

        x_pos, y_pos = 0.02, 0.98
        left_margin = x_pos

        for line_tokens in lines_of_tokens:
            x_pos = left_margin
            for text, color in line_tokens:
                if not text: continue
                ax.text(x_pos, y_pos, text, color=color, fontproperties=font, ha='left', va='top')
                x_pos += len(text) * char_width
            y_pos -= line_height

        ax.axis('off')
        ax.set_ylim(0, 1)
        ax.set_xlim(0, 1)

class Visualize_Helper:
    @staticmethod
    def initialize():
        fig = plt.figure(figsize=(10, 10), layout='constrained')

        gs = gridspec.GridSpec(2, 2, figure=fig)
        skeleton_ax = fig.add_subplot(gs[0, 0], projection='3d')
        mano_ax = fig.add_subplot(gs[0, 1], projection='3d')
        annotation_ax = fig.add_subplot(gs[1, :])
        annotation_ax.axis("off")
        return fig, skeleton_ax, mano_ax, annotation_ax

    @staticmethod
    def format_plain_string(text:str, width:int=90):
        return '\n'.join(textwrap.wrap(text, width=width))


    @staticmethod
    def create_3d_animation(
        left_motion:np.ndarray, right_motion:np.ndarray,
        left_mano:Dict[str, np.ndarray]|None=None, right_mano:Dict[str, np.ndarray]|None=None,
        annotation:str | dict | list="",
        save_path:str|None=None, fps:int=30
    ):
        '''
        left_motion: (T, J, 3)
        right_motion: (T, J, 3)

        left_mano:
            shape: (T, 10)
            pose: (T, 48),
            trans: (T, 3)
        right_mano: the same as left_mano
        '''
        fig, skeleton_ax, mano_ax, annotation_ax = Visualize_Helper.initialize()

        print(f"Annotation Type: {type(annotation)}")

        if isinstance(annotation, dict) or isinstance(annotation, list):
            JsonPlotter.plot_json(annotation, annotation_ax)
        elif isinstance(annotation, str):
            print(f"annotation: {annotation.__repr__()}")
            annotation = '\n'.join(textwrap.wrap(annotation, width=90, replace_whitespace=False, tabsize=4))
            annotation_ax.text(
                x=0.1, y=0.95, s=annotation, ha='left', va='top', fontsize=14
            )
        frame_num = fig.text(
            x=0.5, y=0.95, s="", ha='center', va='top', fontsize=16
        )
        skeleton_helper = Skeleton_Visualize_Helper(skeleton_ax, left_motion, right_motion)
        skeleton_helper.initialize_ax()

        if left_mano is not None and right_mano is not None:
            from .mano2mesh import bihand_mano2mesh
            vertices, faces = bihand_mano2mesh(left_mano, right_mano)
            mesh_helper = Mesh_Visualize_Helper(mano_ax, vertices, faces)
            mesh_helper.initialize_ax()

        def update(frame):
            frame_num.set_text(f"Frame: {frame} / {left_motion.shape[0]}")
            skeleton_helper.draw(frame)
            if left_mano is not None and right_mano is not None:
                mesh_helper.draw_mesh(frame)

        ani = FuncAnimation(fig, update, frames=left_motion.shape[0])

        if save_path:
            pbar = tqdm(total=left_motion.shape[0], desc='Exporting animation')
            ani.save(save_path, writer='ffmpeg', fps=fps, progress_callback=lambda x, y: pbar.update(1))


        plt.close(fig)

class MultiMotionVisualizer:

    def initialize(number_of_motions:int):
        rows = int(math.floor(math.sqrt(number_of_motions)))
        columns = int(math.ceil(number_of_motions / rows))
        fig = plt.figure(figsize=(4 * columns, 4 * rows + 8))
        gs = gridspec.GridSpec(rows + 1, columns, figure=fig, height_ratios=[4] * rows + [8])
        motion_axes = []
        for i in range(number_of_motions):
            ax = fig.add_subplot(gs[i // columns, i % columns], projection='3d')
            motion_axes.append(ax)
        motion_axes = np.array(motion_axes)

        text_ax = fig.add_subplot(gs[rows, :])
        text_ax.axis("off")

        return fig, motion_axes, text_ax

    @staticmethod
    def create_3d_animation(
        motions:List[Dict[str, np.ndarray | str | dict]],
        text:str="",
        save_path:str=None, fps:int=30
    ):
        '''
        if motion['type'] == 'skeleton'
            motion = {
                'type': 'skeleton',
                'left_motion': (T, J, 3),
                'right_motion': (T, J, 3)
            }
        elif motion['type'] == 'mano'
            motion = {
                'type': 'mano',
                'left_motion': dict
                'right_motion': dict'
            }
        '''
        fig, axes, text_ax = MultiMotionVisualizer.initialize(len(motions))

        drawers = []
        frame_num = 0
        for motion_id, motion in enumerate(motions):
            if motion['type'] == 'skeleton':
                helper = Skeleton_Visualize_Helper(
                    axes[motion_id], motion['left_motion'], motion['right_motion'], motion.get('title', None)
                )
                frame_num = max(frame_num, motion['left_motion'].shape[0])
            elif motion['type'] == 'mano':
                from .mano2mesh import bihand_mano2mesh
                vertices, faces = bihand_mano2mesh(
                    motion['left_motion'], motion['right_motion']
                )
                helper = Mesh_Visualize_Helper(
                    axes[motion_id], vertices, faces, motion.get('title', None)
                )
                frame_num = max(vertices.shape[0], frame_num)
            helper.initialize_ax()
            drawers.append(helper)

        frame_num_text = fig.text(
            x=0.5, y=0.95, s="", ha='center', va='top', fontsize=16
        )

        # text = '\n'.join(textwrap.wrap(text, width=90))
        text = smart_wrap(text, width=90)

        text_ax.text(
            x=0.01, y=0.5, s=text, ha='left', va='center', fontsize=14
        )

        def update(frame):
            frame_num_text.set_text(f"Frame: {frame} / {frame_num}")
            for drawer in drawers:
                if isinstance(drawer, Skeleton_Visualize_Helper):
                    drawer.draw(frame)
                elif isinstance(drawer, Mesh_Visualize_Helper):
                    drawer.draw_mesh(frame)

        ani = FuncAnimation(fig, update, frames=frame_num)

        if save_path:
            name = save_path.split('/')[-1]
            pbar = tqdm(total=frame_num, desc=f'Exporting animation to {name}')
            ani.save(save_path, writer='ffmpeg', fps=fps, progress_callback=lambda x, y: pbar.update(1))

        plt.close(fig)