Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Roboto-VariableFont_wdth,wght.ttf filter=lfs diff=lfs merge=lfs -text
+SkeletonDiffusion_Demo/magick filter=lfs diff=lfs merge=lfs -text
+magick filter=lfs diff=lfs merge=lfs -text
+models/nlf_l_multi.torchscript filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,175 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# PyPI configuration file
+.pypirc
+
+9622_GRAB/
+
+*.pkl

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

README.md

@@ -1,12 +1,13 @@
 ---
 title: SkeletonDiffusion
-emoji: 📈
-colorFrom: green
-colorTo: pink
+emoji: 💻
+colorFrom: purple
+colorTo: green
 sdk: gradio
-sdk_version: 5.20.0
+sdk_version: 5.12.0
 app_file: app.py
 pinned: false
+license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

Roboto-VariableFont_wdth,wght.ttf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b25986d18730960c7b27384ab2bc500856ae7fe9e71c9850019195ff9019f0b2
+size 468308

SkeletonDiffusion_Demo/.gitignore

@@ -0,0 +1,173 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# PyPI configuration file
+.pypirc
+
+test/

SkeletonDiffusion_Demo/README.md

@@ -0,0 +1,52 @@
+# Generate gifs from sequences of obj files
+## Environment
+Sorry, there is no environment file (the env I am currently using contains packages that are not necessary here). Here the version of a few necessary modules. You will have to install other modules as warning appears once you try running the code.
+```
+vedo                      2024.5.1     
+vpython                   7.6.5 
+python                    3.10.13 
+```
+
+## Run the code
+### Prepare the data
+Copy the directory '/usr/wiss/curreli/work/my_modules/Skeleton2Mesh/final_output/hmp/visuals_50samples/amass/SkeletonDiffusion/test' to this folder (I already gave you the rights - afterwards I will delete it). There are meshes for two test segments from AMASS, 9622_GRAB and 11949_HUMAN4D. The predictions are named according to the following convention:
+ - obs: Past obervation (input sequence)
+ - gt: GT future
+ - pred_closest_GT: prediction that is most similar to the GT
+ - most_far_<idx>: the remaining predictions are sorted accordng to their similarity to the GT future, or better to the similarity to the prediction closest to the GT. So most_far_0 is the prediction most diverse from the GT
+
+In the demo we will not have a GT future. We will order the predictions according to a different criterion (maybe diversity).
+
+### Generate visuals
+Now you can run
+```
+python plot_several_meshes.py --mesh_parent_folder=<put_correct_path_here/9622_GRAB>
+```
+The output should be in a parallel direction at lowest folder level.
+ ## If you are using a ssh connection
+ You probably have no screen, so the code terminates. You can avoid the problem by generating a screen first with our script  setup_headless.bash: 
+ ```
+ setup_headless.bash python plot_several_meshes.py <...>
+ ```
+ ### Debug via ssh connection
+ OPen the file of your VS Studio debug configurations, launch.json, and add this new configuration:
+  
+```
+    { "name": "Python: Attach to Edge", 
+    "request": "attach", "type": "python", 
+    "connect": { "host": "localhost", "port": 5678 },
+    "justMyCode": true,
+    },
+```
+Now start the file in debugger mode with debugpy
+```
+./setup_headless.bash python -m debugpy --listen 5678 --wait-for-client plot_several_meshes.py
+ ```
+
+Now start your debug configuraiton ""Python: Attach to Edge" to attach your debugger and start debugging.
+
+## For the future 
+If you need to put more functions together consider using shell cripts
+```
+bash create_mesh_gifs.sh
+```

SkeletonDiffusion_Demo/combine_video.py

@@ -0,0 +1,77 @@
+# import tempfile
+# from moviepy import *
+
+# def combine_video(sample_obj_path):
+#     gif1 = VideoFileClip(sample_obj_path+ "/shadow_gif/gt_obj_tranp.gif")
+#     gif2 = VideoFileClip(sample_obj_path+ "/shadow_gif/pred_closest_GT_obj_tranp.gif")
+#     gif3 = VideoFileClip(sample_obj_path+ "/shadow_gif/pred_far_0_obj_tranp.gif")
+
+#     common_duration = min(gif1.duration, gif2.duration, gif3.duration)
+#     gif1 = gif1.subclipped(0, common_duration)
+#     gif2 = gif2.subclipped(0, common_duration)
+#     gif3 = gif3.subclipped(0, common_duration)
+#     print(gif1.duration, gif2.duration, gif3.duration)
+
+#     text1 = TextClip(font = 'Roboto-VariableFont_wdth,wght.ttf', text="Ground Truth", font_size=30, color='black').with_position('center').with_duration(common_duration)
+#     text2 = TextClip(font = 'Roboto-VariableFont_wdth,wght.ttf', text="Generated_1", font_size=30, color='black').with_position('center').with_duration(common_duration)
+#     text3 = TextClip(font = 'Roboto-VariableFont_wdth,wght.ttf', text="Generated_2", font_size=30, color='black').with_position('center').with_duration(common_duration)
+
+#     gif1_with_text = CompositeVideoClip([gif1, text1.with_position(("center", "top"))])
+#     gif2_with_text = CompositeVideoClip([gif2, text2.with_position(("center", "top"))])
+#     gif3_with_text = CompositeVideoClip([gif3, text3.with_position(("center", "top"))])
+
+#     combined = clips_array([[gif1_with_text, gif2_with_text, gif3_with_text]])
+#     # combined = clips_array([[gif1, gif2, gif3]])
+
+#     target_duration = 2.5
+#     original_duration = combined.duration
+#     speed_factor = original_duration / target_duration
+
+#     accelerated_video = combined.with_speed_scaled(factor=speed_factor*3)
+
+#     # accelerated_video.write_videofile("combined_video.mp4", fps=60)
+#     output_path = tempfile.NamedTemporaryFile(suffix=".gif", delete=False).name
+#     accelerated_video.write_gif(output_path, fps=60)
+#     return output_path
+
+# if __name__ == "__main__":
+#     combine_video("./9622_GRAB/")
+
+import tempfile, os
+from moviepy.editor import VideoFileClip, TextClip, CompositeVideoClip, clips_array, vfx
+
+def combine_video(sample_obj_path):
+    # os.environ['IMAGEMAGICK_BINARY'] = '/home/stud/yaji/storage/user/yaji/NonisotropicSkeletonDiffusion/magick'
+    gif1 = VideoFileClip(sample_obj_path+ "/shadow_gif/gt_obj_tranp.gif")
+    gif2 = VideoFileClip(sample_obj_path+ "/shadow_gif/pred_closest_GT_obj_tranp.gif")
+    gif3 = VideoFileClip(sample_obj_path+ "/shadow_gif/pred_far_0_obj_tranp.gif")
+
+    common_duration = min(gif1.duration, gif2.duration, gif3.duration)
+    gif1 = gif1.subclip(0, common_duration)
+    gif2 = gif2.subclip(0, common_duration)
+    gif3 = gif3.subclip(0, common_duration)
+    print(gif1.duration, gif2.duration, gif3.duration)
+
+    text1 = TextClip("Ground Truth", fontsize=30, color='black').set_position('center').set_duration(common_duration)
+    text2 = TextClip("Generated_1", fontsize=30, color='black').set_position('center').set_duration(common_duration)
+    text3 = TextClip("Generated_2", fontsize=30, color='black').set_position('center').set_duration(common_duration)
+
+    gif1_with_text = CompositeVideoClip([gif1, text1.set_position(("center", "top"))])
+    gif2_with_text = CompositeVideoClip([gif2, text2.set_position(("center", "top"))])
+    gif3_with_text = CompositeVideoClip([gif3, text3.set_position(("center", "top"))])
+
+    combined = clips_array([[gif1_with_text, gif2_with_text, gif3_with_text]])
+    # combined = clips_array([[gif1, gif2, gif3]])
+
+    target_duration = 2.5
+    original_duration = combined.duration
+    speed_factor = original_duration / target_duration
+
+    accelerated_video = combined.fx(vfx.speedx, factor=speed_factor)
+
+    output_path = tempfile.NamedTemporaryFile(suffix=".gif", delete=False).name
+    accelerated_video.write_gif(output_path, fps=60)
+    return output_path
+
+if __name__ == "__main__":
+    combine_video("./9622_GRAB/")

SkeletonDiffusion_Demo/create_mesh_gifs.sh

@@ -0,0 +1,3 @@
+export IMAGEMAGICK_BINARY=/home/stud/yaji/storage/user/yaji/SkeletonDiffusion_Demo/magick
+
+python plot_several_meshes.py -f <you_custom_path>

SkeletonDiffusion_Demo/magick

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:945eaba92f0d94fb4d54b45cfd67c2bdd163b987d66c6ca108d470b09049bfe0
+size 29795520

SkeletonDiffusion_Demo/plot_several_meshes.py

@@ -0,0 +1,304 @@
+import numpy as np
+import cv2
+from vedo import Mesh, Plane, Plotter, screenshot
+import imageio.v2 as imageio
+# import imageio as imageio
+import re
+import os
+from .visual import get_ptc # probaly vpython
+from PIL import Image
+from tqdm import tqdm
+
+screenshot_scale = 3
+
+PLAY_OBS_BEFORE_PRED = True
+obs_dir = ""
+cam = dict(
+        position=(-3.85922, -4.78140, 2.689328),
+        focal_point=(0.192544, 4.77379e-3, 0.0127248),
+        viewup=(0.0724348, 0.109097, 0.991388),
+        distance=5.25119,
+        clipping_range=(4.38228, 6.36775),
+    )
+
+_green= [0.4, 0.7, 0.5]
+_blue =[0.7, 0.9, 1.0]
+_dark_blue =[0.03, 0.4, 0.7]
+_orange = [0.9, 0.7, 0.2]
+
+def get_mesh(mesh_name, color='darksalmon', mesh_opacity=1.0, mesh_lighting='default', rotation=[0, 0, 0], offset=[0, 0, 0], scale=0):
+    offset = np.array(offset)
+    #styles = ['default', 'metallic', 'plastic', 'shiny', 'glossy', 'ambient', 'off']
+    mesh = Mesh(mesh_name, c=color, alpha=mesh_opacity).lighting(mesh_lighting)
+    if scale == 0:
+        bnd = np.array(mesh.bounds())
+        scale = 1 / max(bnd[1]-bnd[0], bnd[3]-bnd[2], bnd[5]-bnd[4])
+    mesh.scale(scale)
+    mesh.pos(offset)
+    mesh.rotate_x(rotation[0])
+    mesh.rotate_y(rotation[1])
+    mesh.rotate_z(rotation[2])
+    # mesh.phong()
+    # mesh2.phong()
+    return mesh, scale
+
+def save_png_rm_bg(filename, im, bg_color_min=[0, 255, 100], bg_color_max=None, rgb=False):
+    # https://stackoverflow.com/questions/72062001/remove-everything-of-a-specific-color-with-a-color-variation-tolerance-from-an
+    # for transperancy:
+    # https://stackoverflow.com/questions/55582117/efficiently-converting-color-to-transparency-in-python
+
+    if bg_color_max is None:
+        bg_color_max = bg_color_min
+    # bg_color_min = [bg_color_min[2], bg_color_min[1], bg_color_min[0]]
+    # bg_color_max = [bg_color_max[2], bg_color_max[1], bg_color_max[0]]
+    # Load image
+    # im = cv2.imread('test.png')
+
+    # Define lower and upper limits of our blue
+    min = np.array(bg_color_min, np.uint8)
+    max = np.array(bg_color_max, np.uint8)
+
+    # Go to HSV colourspace and get mask of blue pixels
+    # HSV = cv2.cvtColor(im, cv2.COLOR_BGR2HSV)
+    mask = cv2.inRange(im, min, max)
+    # Try dilating (enlarging) mask with 3x3 structuring element
+    # SE = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
+    # mask = cv2.dilate(mask, kernel, iterations=1)
+
+    # create alpha channel
+    alpha = (im[:, :, 0] * 0 + 255)
+
+    # Make all pixels in mask white
+    alpha[mask > 0] = 0
+
+    # add alpha channel
+    if rgb:
+        im = cv2.merge((im[:, :, 0], im[:, :, 1], im[:, :, 2], alpha))
+    else:
+        im = cv2.merge((im[:, :, 2], im[:, :, 1], im[:, :, 0], alpha))
+
+    # crop to visible part
+    mask = im[:, :, 3] != 0.
+    coords = np.argwhere(mask)
+    y0, x0 = coords.min(axis=0)
+    y1, x1 = coords.max(axis=0) + 1
+    # im = im[y0:y1, x0:x1, :]
+
+    # write to file
+    cv2.imwrite(filename, im)
+
+def save_plot_as_transparent_png(plt, out_filename):
+    # if not out_filename is None:
+    bg_col = [255, 255, 255]
+    plt.background(c1=bg_col)
+    save_png_rm_bg(out_filename, plt.screenshot(asarray=True), bg_color_min=bg_col)
+
+
+def get_mesh_shadow(mesh, offset=[0, 0, 0], plane_normal=(0, 0, 1), direction=[0.1, -1.8, 3]):
+    shadow = []
+    shad_col = np.array([0.8, 0.8, 0.8])
+    min_z = mesh.vertices.min(axis=0)[2]
+    # mesh.add_shadow(plane='z', point=min_z, alpha=1, c=shad_col, culling=0.9,)
+    plane = Plane(pos=np.array([0, 0, min_z]) + np.array(offset), normal=plane_normal, s=[7, 7]).alpha(0.2)
+    shad = mesh.clone().project_on_plane(plane, direction=-np.array(direction) + np.array(offset))
+    shad.c(shad_col).alpha(1).lighting("off").use_bounds(False)
+    shadow = shad
+    return shadow
+
+def plot_meshes(mesh_files, cam, mesh_color=[1, 1, 1], rotation=[0, 0, 0]):
+    
+    mesh_opacity = 0.8
+    plot_shadows = True
+    
+    pass
+
+def create_visual(mesh_files, color='coral', dest_name="", x=0,y=0,z=0, type='mesh', mesh_lighting='plastic', obs_dir=""):
+    meshes, scales =[], [] 
+    bg_col = [255, 255, 255]# plotter = Plotter(offscreen=True)
+    plotter = Plotter(bg=[255, 255, 255], offscreen=True)
+
+    frame_files = []
+    # lights = get_lights()
+    for file in tqdm(mesh_files):
+        mesh, scale = get_mesh(file, color=color, rotation=[x,y,z], mesh_lighting=mesh_lighting, scale=0.6)
+        # scalars = 1- mesh.vertices[:, 2]
+        # mesh.cmap('Oranges', scalars)
+       
+        plotter.clear()
+        
+        shadow = get_mesh_shadow(mesh)
+        
+        # plotter.add([mesh, shadow])
+        # plotter.show(mesh, __doc__, interactive=False, camera=cam, resetcam=True, zoom=2)
+        plotter.show(mesh, shadow, __doc__, interactive=False, camera=cam, resetcam=True, zoom=2)
+        frame_path =file[:-4] + ".png"
+        frame_files.append(frame_path)
+        screenshot(frame_path, scale=1)
+        save_png_rm_bg(file[:-4]+"new.png", plotter.screenshot(asarray=True), bg_color_min=bg_col)
+
+    # Compile saved frames into a GIF
+    filename = os.path.join(dest_name + '.gif')
+    create_transparent_gif(frame_files, dest_name, obs_dir)
+    # with imageio.get_writer(filename, mode='I', duration=2) as writer: #duration=0.2) as writer:
+    #     for filename in frame_files:
+    #         image = imageio.imread(filename)
+    #         writer.append_data(image)
+
+
+def create_transparent_gif(frame_files, dest_name, obs_dir):
+    images = []
+    if "obs" not in dest_name and PLAY_OBS_BEFORE_PRED:
+        # visit all file under obs_dir that end with new.png
+        obs_files = [os.path.join(obs_dir, f) for f in os.listdir(obs_dir) if f.endswith('new.png')]
+        # sort the files
+        obs_files = sort_list(obs_files)
+        for i, frame in enumerate(obs_files):
+            im = Image.open(frame)
+            images.append(im)
+    for i, frame in enumerate(frame_files):
+        im = Image.open(frame.replace('.png', 'new.png'))
+        images.append(im)
+
+    # print(images.size)
+    images[0].save(
+            os.path.join(dest_name + '_tranp.gif'),
+            format="GIF",
+            save_all=True,
+            loop=1,
+            append_images=images,
+            duration=0.016,
+            disposal=2,
+    )    
+
+def sort_list(l):
+    try:
+        return list(sorted(l, key=lambda x: int(re.search(r'\d+(?=\.)', x).group())))
+    except AttributeError:
+        return sorted(l)
+    
+
+def visual_gt(mesh_files, color, x,y,z, type='mesh'):
+    bg_col = [255, 255, 255]# plotter = Plotter(offscreen=True)
+    plotter = Plotter(bg=[255, 255, 255], offscreen=True)
+
+    frame_files = []
+    # lights = get_lights()
+    for file in mesh_files:
+        
+        if type == 'mesh':
+            mesh, scale = get_mesh(file, color=color, rotation=[x,y,z], mesh_lighting='plastic')
+            # scalars = 1- mesh.vertices[:, 2]
+            # mesh.cmap('Blues', scalars)
+        else:
+            mesh, scale = get_ptc(file, rotation=[x,y,z])
+            mesh.c([0.5,0.5,0.5])
+       
+        plotter.clear()
+        
+        shadow = get_mesh_shadow(mesh)
+        
+        # plotter.add([mesh, shadow])
+        plotter.show(mesh, shadow, __doc__, interactive=False, camera=cam, resetcam=True, zoom=2)
+        frame_path =file[:-4] + ".png"
+        frame_files.append(frame_path)
+        screenshot(frame_path, scale=1)
+        save_png_rm_bg(file[:-4]+"new.png", plotter.screenshot(asarray=True), bg_color_min=bg_col)
+
+
+def plot_two_mesh(meshfile1, meshfile2, color, x,y,z):
+    plotter = Plotter(bg=[255, 255, 255], offscreen=True)
+    mesh1, scale = get_ptc(meshfile1, rotation=[x,y,z])
+    mesh1.alpha(1.0).c([0.5,0.5,0.5])
+    # scalars = 1- mesh1.vertices[:, 2]
+    # mesh1.cmap('Oranges', scalars)
+    # mesh1.c(_dark_blue)
+    mesh2, scale = get_mesh(meshfile2, color=color, rotation=[x,y,z], mesh_lighting='plastic')
+   
+    shadow = get_mesh_shadow(mesh2)
+    plotter.show(mesh1, mesh2, shadow, __doc__, interactive=False, camera=cam, resetcam=True, zoom=2)
+    frame_path =meshfile2[:-4] + ".png"
+    
+    screenshot(frame_path, scale=1)
+    save_png_rm_bg(meshfile2[:-4]+"both.png", plotter.screenshot(asarray=True), bg_color_min=[255, 255, 255])
+    
+import argparse
+
+  
+    
+
+def main(parent_folder):
+    file_type = '.obj' #'mesh.ply'
+    x,y,z = 0, 0, 0
+    dest_dir = os.path.join(os.path.dirname(parent_folder), 'shadow_gif')
+    obs_dir = os.path.join(os.path.dirname(parent_folder), 'obs_obj')
+    os.makedirs(dest_dir, exist_ok=True)
+    subfolders = sorted([f for f in os.listdir(parent_folder) if os.path.isdir(os.path.join(parent_folder, f)) and 'obj' in f and ('gt' in f or 'closest' in f or 'far_49' in f)])
+    for f in subfolders:
+        folder = os.path.join(parent_folder, f) #'final_output/hmp/visuals_10samples/h36m/SkeletonDiffusion/test/952_WalkTogether/obj/gt'
+        dest_name = os.path.join(dest_dir, f) #'./meshes_with_shadows_aspng'
+
+
+        mesh_files = [os.path.join(folder, f) for f in os.listdir(folder) if f.endswith(file_type)]
+        mesh_files = sort_list(mesh_files)
+        os.makedirs(dest_dir, exist_ok=True)
+
+        
+        #color "lightblue" "navajowhite"
+        if  'obs'  in f:
+            create_visual(mesh_files, _green, dest_name, x,y,z, type='mesh', mesh_lighting='plastic', obs_dir=obs_dir) #--> mesh_animation.gif
+        else: 
+            create_visual(mesh_files, _dark_blue, dest_name, x,y,z, type='mesh', mesh_lighting='plastic', obs_dir=obs_dir) #--> mesh_animation.gif
+    
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    #conda activate hmp_visualize
+    parser.add_argument('-f', '--mesh_parent_folder', type=str, default="final_output/hmp/visuals/amass/SkeletonDiffusion/test/952_WalkTogether/obj")
+    
+    args = parser.parse_args()
+    main(args.mesh_parent_folder)
+    
+    ####################################################################################################################################
+    # Old code snippet, kept just for reference. Jialin here there may be somehting you will need, or maybe not :) If you do not need you can delete
+    ####################################################################################################################################
+
+    # other functions not needed, from older code
+    # visual_gt(gt_mesh_files, 0,0,-90, type='mesh') # this line was commented out by Lu
+    # visual_gt(gt_obj_files, 'salmon', x,y,z)#--> .png and new.png (transparent) per frame
+    # gt_ptc_files = [os.path.join(folder, prefix +'gt_pointcloud_x.ply'),os.path.join(folder, prefix +'gt_pointcloud_y.ply')]
+    # gt_obj_files = sort_list([os.path.join(target_folder, f) for f in os.listdir(target_folder) if f.endswith(file_type)])
+    # plot_two_mesh(gt_ptc_files[1], mesh_files[-1], "lightblue", x,y,z)
+
+    # older code
+    # final_folder
+    # gif_folder
+    # filename = os.path.join(dest_name + '.gif')
+    # frame_files = [os.path.join(gif_folder, f"pred{i}.gif") for i in range(10)]
+    # obs_image = imageio.imread(os.path.join(gif_folder, f"obs_salmon.png"))
+    # for i,filename in enumerate(frame_files):
+    #     image = imageio.imread(filename)
+    #     frames = np.vstack([obs_image, image])
+    #     imageio.imwrite(os.path.join(final_folder, f"pred{i}.gif"), frames, fps=50)
+
+    # image = imageio.imread(os.path.join(gif_folder, f"gt_salmon.gif"))
+    # frames = np.vstack([obs_image, image])
+    # imageio.imwrite(os.path.join(final_folder, f"gt.gif"), frames, fps=50)
+
+
+
+    # gifs= [imageio.imread(os.path.join(final_folder, f"pred{i}.gif")) for i in range(10)]
+    # #Create writer object
+    # new_gif = imageio.get_writer(os.path.join(final_folder, f"output.gif"))
+
+    # for frame_number in range(len(gifs[0])):
+    #     img1 = gif1.get_next_data()
+    #     img2 = gif2.get_next_data()
+    #     #here is the magic
+    #     upper_row = np.hstack((gifs[:5, frame_number]))
+    #     lower_row = np.hstack((gifs[5:, frame_number]))
+    #     new_image = np.vstack((upper_row, lower_row))
+    #     new_gif.append_data(new_image)
+
+
+    print("done.")
+

SkeletonDiffusion_Demo/setup_headless.bash

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -x
+export DISPLAY=:99.0
+which Xvfb
+Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
+# sleep 3
+set +x
+exec "$@"

SkeletonDiffusion_Demo/visual.py

@@ -0,0 +1,322 @@
+from vedo import Mesh, Points, Plotter, screenshot, Arrow, Light, Plane
+import imageio
+import os
+import numpy as np
+from scipy.interpolate import Rbf, NearestNDInterpolator as Near
+import re
+
+screenshot_scale = 3
+
+cam = dict(
+        position=(-3.85922, -4.78140, 0.689328),
+        focal_point=(0.192544, 4.77379e-3, 0.0127248),
+        viewup=(0.0724348, 0.109097, 0.991388),
+        distance=5.25119,
+        clipping_range=(4.38228, 6.36775),
+    )
+
+_red = [0.9, 0.1, 0.1]
+_green= [0.4, 1.0, 0.8]
+_blue =[0.7, 0.9, 1.0]
+_dark_blue =[0.03, 0.4, 0.7]
+
+def sort_list(l):
+    try:
+        return list(sorted(l, key=lambda x: int(re.search(r'\d+(?=\.)', x).group())))
+    except AttributeError:
+        return sorted(l)
+
+
+def get_lights(light_color=[255, 255, 245], offset=[0, 0, 0], intensity_mult=1):
+    # Add light sources at the given positions
+    # (grab the position and color of the arrow object)
+    orig = np.array([0, 0, 0]) + np.array(offset)
+    phl = Arrow(np.array([0.1, 0.1, 10]) + offset, orig, c=light_color).scale(0.2)
+    pfl = Arrow(np.array([1.5, 0.1, 0.3]) + offset, orig, c=light_color).scale(0.2)
+    pbl = Arrow(np.array([-1.5, 0.1, 0.3]) + offset, orig, c=light_color).scale(0.2)
+    prl = Arrow(np.array([0.1, -1.5, 0.3]) + offset, orig, c=light_color).scale(0.2)
+    pll = Arrow(np.array([0.1, 1.5, 0.3]) + offset, orig, c=light_color).scale(0.2)
+    hl = Light(phl, intensity=0.7 * intensity_mult, angle=180, )
+    rl = Light(pfl, intensity=0.6 * intensity_mult, angle=180, )
+    ll = Light(pbl, intensity=0.6 * intensity_mult, angle=180, )
+    bl = Light(pll, intensity=0.6 * intensity_mult, angle=180, )
+    fl = Light(prl, intensity=1 * intensity_mult, angle=180, )
+    lights = [hl, fl, bl, ll, rl]
+    return lights
+
+def get_mesh(mesh_name, mesh_opacity=1.0, mesh_lighting='default', rotation=[0, 0, 0], offset=[0, 0, 0], scale=0):
+    offset = np.array(offset)
+    #styles = ['default', 'metallic', 'plastic', 'shiny', 'glossy', 'ambient', 'off']
+    mesh = Mesh(mesh_name, c=_blue, alpha=mesh_opacity).lighting(mesh_lighting)
+    if scale == 0:
+        bnd = np.array(mesh.bounds())
+        scale = 1 / max(bnd[1]-bnd[0], bnd[3]-bnd[2], bnd[5]-bnd[4])
+    mesh.scale(scale)
+    mesh.pos(offset)
+    mesh.rotate_x(rotation[0])
+    mesh.rotate_y(rotation[1])
+    mesh.rotate_z(rotation[2])
+    # mesh.phong()
+    # mesh2.phong()
+    return mesh, scale
+
+def get_ptc(ptc_name, rotation=[0, 0, 0], offset=[0, 0, 0], scale=0, color=_dark_blue):
+    offset = np.array(offset)
+    #styles = ['default', 'metallic', 'plastic', 'shiny', 'glossy', 'ambient', 'off']
+    mesh = Points(Mesh(ptc_name), r=1).c([0.5,0.5,0.5]).alpha(0.8)
+    if scale == 0:
+        bnd = np.array(mesh.bounds())
+        scale = 1 / max(bnd[1]-bnd[0], bnd[3]-bnd[2], bnd[5]-bnd[4])
+    mesh.scale(scale)
+    mesh.pos(offset)
+    mesh.rotate_x(rotation[0])
+    mesh.rotate_y(rotation[1])
+    mesh.rotate_z(rotation[2])
+    # mesh.phong()
+    # mesh2.phong()
+    return mesh, scale
+
+def get_mesh_shadow(mesh, offset=[0, 0, 0], plane_normal=(0, 0, 1), direction=[0.1, -1.8, 3]):
+    shadow = []
+    shad_col = np.array([0.8, 0.8, 0.8])
+    min_z = mesh.points().min(axis=0)[2]
+    # mesh.add_shadow(plane='z', point=min_z, alpha=1, c=shad_col, culling=0.9,)
+    plane = Plane(pos=np.array([0, 0, min_z]) + np.array(offset), normal=plane_normal, s=[7, 7]).alpha(0.2)
+    shad = mesh.clone().project_on_plane(plane, direction=-np.array(direction) + np.array(offset))
+    shad.c(shad_col).alpha(1).lighting("off").use_bounds(False)
+    shadow = shad
+    return shadow
+
+
+
+
+def create_visual(mesh_files, directory, x,y,z, type='mesh'):
+    meshes, scales =[], []
+    # plotter = Plotter(offscreen=True)
+    plotter = Plotter(bg=[255, 255, 255], offscreen=True)
+
+    frame_files = []
+    # lights = get_lights()
+    for file in mesh_files:
+        if type == 'mesh':
+            mesh, scale = get_mesh(file, rotation=[x,y,z], mesh_lighting='metallic')
+        else:
+            mesh, scale = get_ptc(file)
+        plotter.clear()
+        
+        shadow = get_mesh_shadow(mesh)
+        # plotter.add([mesh, shadow])
+        plotter.show(mesh, shadow, __doc__, interactive=False, camera=cam, resetcam=True, zoom=2)
+        frame_path =file[:-4] + ".png"
+        frame_files.append(frame_path)
+        screenshot(frame_path, scale=1)
+    
+    # Compile saved frames into a GIF
+    filename = os.path.join(directory, type+'_animation.gif')
+    with imageio.get_writer(filename, mode='I', duration=0.2) as writer:
+        for filename in frame_files:
+            image = imageio.imread(filename)
+            writer.append_data(image)
+            
+        
+
+def create_mesh_animation(mesh_files, directory, x,y,z):
+    
+    # Load all meshes
+    meshes = [Mesh(f) for f in mesh_files]
+
+    # Create a plotter
+    plotter = Plotter(bg=[255, 255, 255], offscreen=True)
+    # plotter = vedo.Plotter()# Use offscreen rendering to save frames
+
+    frame_files = []  # To store the path of each frame image
+    
+    # plotter.camera.SetViewUp(0,0,1)
+    
+    
+    # shadow = mesh.clone().projectOnPlane(direction=(0,0,-1))
+
+    # Color the shadow mesh dark and make it slightly transparent
+    
+    for frame_id, mesh in enumerate(meshes):
+        mesh.rotate_x(x).rotate_y(y).rotate_z(z).lighting('metallic')
+        shadow = get_mesh_shadow(mesh)
+        scalars = 1- mesh.vertices[:, 2]
+        mesh.cmap('Blues', scalars)
+        # mesh.add_shadow('x', 0.95)
+        plotter.clear()
+        # plotter.add(mesh)
+        plotter.show(mesh, shadow, __doc__, interactive=False, camera=cam, resetcam=True, zoom=2)
+        frame_path = mesh_files[frame_id][:-4] + ".png"
+        frame_files.append(frame_path)
+        screenshot(frame_path, scale=1)  # Save frame as image
+        
+        
+        #crop the images
+        # os.system("""convert -trim {0} "{1}" """.format(frame_path, frame_path))
+
+
+    # Compile saved frames into a GIF
+    filename = os.path.join(directory, 'mesh_animation.gif')
+    with imageio.get_writer(filename, mode='I', duration=0.2) as writer:
+        for f in frame_files:
+            image = imageio.imread(f)
+            writer.append_data(image)
+            os.system("""convert -trim {0} "{1}" """.format(f, f))
+    
+    
+
+
+
+def create_point_cloud_animation(files, directory, x,y,z):
+    # Load all meshes
+    meshes = [Points(Mesh(f), r=1) for f in files]
+
+    # Create a plotter
+    plotter = Plotter(bg=[255, 255, 255], offscreen=True)
+    # plotter = vedo.Plotter()# Use offscreen rendering to save frames
+
+    frame_files = []  # To store the path of each frame image
+    
+    # plotter.camera.SetViewUp(0,0,1)
+    
+    
+    # shadow = mesh.clone().projectOnPlane(direction=(0,0,-1))
+
+    # Color the shadow mesh dark and make it slightly transparent
+    
+    for frame_id, mesh in enumerate(meshes):
+        mesh.rotate_x(x).rotate_y(y).rotate_z(z).c([0.5,0.5,0.5])
+        shadow = get_mesh_shadow(mesh)
+        # scalars = 1- mesh.vertices[:, 2]
+        # mesh.cmap('binary', scalars)
+        plotter.clear()
+        plotter.show(mesh, shadow, __doc__, interactive=False, camera=cam, resetcam=True, zoom=2)
+
+        frame_path = files[frame_id][:-4] + ".png"
+        frame_files.append(frame_path)
+        screenshot(frame_path, scale=1)  # Save frame as image
+        
+        
+        #crop the images
+        # os.system("""convert -trim {0} "{1}" """.format(frame_path, frame_path))
+
+
+    # Compile saved frames into a GIF
+    filename = os.path.join(directory, 'pct_animation.gif')
+    with imageio.get_writer(filename, mode='I', duration=0.2) as writer:
+        for f in frame_files:
+            image = imageio.imread(f)
+            writer.append_data(image)
+            os.system("""convert -trim {0} "{1}" """.format(f, f))
+
+
+
+def point_cloud_colormap(gt_ptc, ptc, corr_num):
+    
+    s1, _ = get_ptc(gt_ptc, color=_red)
+    s2, _ = get_ptc(ptc, color=_blue)
+    
+    landmarks1 = s1.vertices[:corr_num,:]
+    landmarks2 = s2.vertices[:corr_num,:]
+    
+    x, y, z = np.split(landmarks1, 3, axis=1)
+    desc = x**2 + y**2 + z**2
+    
+    # itr = Rbf(x, y, z, desc)          # Radial Basis Function interpolator
+    itr = Near(landmarks1, desc)       # Nearest-neighbour interpolator
+
+    # interpolate descriptor on the full set of mesh vertices
+    xi, yi, zi = np.split(s2.vertices, 3, axis=1)
+    interpolated_desc = itr(xi, yi, zi)
+    
+    s2.cmap('coolwarm', interpolated_desc.squeeze()).add_scalarbar(title='error')
+    # s2.point_colors(interpolated_desc, cmap='coolwarm', vmin=min(interpolated_desc), vmax=max(interpolated_desc)).addScalarBar(title='$\error$')
+    
+    plotter = Plotter()
+    
+    plotter.show(s2, __doc__, axes=True)
+
+
+def mesh_colormap(gt_mesh, mesh, corr_num):
+    
+    s1, _ = get_ptc(gt_ptc, color=_red)
+    s2, _ = get_ptc(ptc, color=_blue)
+    
+    landmarks1 = s1.vertices[:corr_num,:]
+    landmarks2 = s2.vertices[:corr_num,:]
+    
+    x, y, z = np.split(landmarks1, 3, axis=1)
+    desc = x**2 + y**2 + z**2
+    
+    # itr = Rbf(x, y, z, desc)          # Radial Basis Function interpolator
+    itr = Near(landmarks1, desc)       # Nearest-neighbour interpolator
+
+    # interpolate descriptor on the full set of mesh vertices
+    xi, yi, zi = np.split(s2.vertices, 3, axis=1)
+    interpolated_desc = itr(xi, yi, zi)
+    
+    s2.cmap('coolwarm', interpolated_desc.squeeze()).add_scalarbar(title='error')
+    # s2.point_colors(interpolated_desc, cmap='coolwarm', vmin=min(interpolated_desc), vmax=max(interpolated_desc)).addScalarBar(title='$\error$')
+    
+    plotter = Plotter()
+    
+    plotter.show(s2, __doc__, axes=True)
+    
+    
+def plot_two_mesh(meshfile1, meshfile2, x,y,z):
+    plotter = Plotter(bg=[255, 255, 255], offscreen=True)
+    mesh1, scale = get_ptc(meshfile1, rotation=[x,y,z])
+    mesh1.alpha(1.0).c([0.2,0.2,0.2])
+    # scalars = 1- mesh1.vertices[:, 2]
+    # mesh1.cmap('Oranges', scalars)
+    # mesh1.c(_dark_blue)
+    mesh2, scale = get_mesh(meshfile2, rotation=[x,y,z], mesh_lighting='shiny')
+    scalars = 1- mesh2.vertices[:, 2]
+    mesh2.cmap('Blues', scalars)
+    plotter.show(mesh1, mesh2, __doc__, interactive=False, camera=cam, resetcam=True, zoom=2)
+    frame_path =meshfile2[:-4] + ".png"
+    
+    screenshot(frame_path, scale=1)
+    os.system("""convert -trim {0} "{1}" """.format(frame_path, frame_path))
+    
+
+if __name__ == "__main__":
+    
+    folder = '/home/wiss/sang/git/implicit_neuro_morph/jax_implicit_neuro_morph/exp/smal/2024_04_18_15_11_59/reconstructions'
+    prefix = 'shape_0_to_4000_'
+    
+    # filenames = [os.path.join(folder, prefix + str(i) + '_mesh.ply') for i in range(0,11)]
+    # create_mesh_animation(filenames, folder, -45, -45, 90)
+    
+    
+    # filenames = [os.path.join(folder, prefix + str(i) + '_ptc.ply') for i in range(0,11)]
+    
+    # gt_ptc = os.path.join(folder, prefix + 'gt_pointcloud_y.ply')
+    # ptc = os.path.join(folder, prefix + 'epoch_10000_time_10_ptc.ply')
+    
+    # create_point_cloud_animation(filenames, folder, -45, -45, 90)
+    
+    folder = '/home/wiss/sang/git/implicit_neuro_morph/jax_implicit_neuro_morph/exp/smal/2024_04_18_15_11_59/eval'
+    # mesh_files = [os.path.join(folder, f) for f in os.listdir(folder) if f.endswith('mesh.ply')]
+    # mesh_files = sort_list(mesh_files)
+    
+    # create_visual(mesh_files, folder, 0,0,-90, type='mesh')
+    
+    # create_mesh_animation(mesh_files, folder, 0,0, -90)
+    
+    #  folder = '/home/wiss/sang/git/implicit_neuro_morph/jax_implicit_neuro_morph/exp/fraust_r/2024_04_22_14_01_05/eval'
+    # mesh_files = [os.path.join(folder, f) for f in os.listdir(folder) if f.endswith('ptc.ply')]
+    # mesh_files = sort_list(mesh_files)
+    # create_point_cloud_animation(mesh_files, folder, 0,0,-90)
+    
+    prefix = 'shape_0_to_4000_'
+    mesh1 = 'gt_pointcloud_y.ply'
+    mesh2 = 'step_10_mesh.ply'
+    
+    meshfile1 = os.path.join(folder, prefix + mesh1)
+    meshfile2 = os.path.join(folder, prefix + mesh2)
+    
+    plot_two_mesh(meshfile1, meshfile2, 0, 0, -30)
+    
+    

app.py

@@ -0,0 +1,264 @@
+import pathlib
+import gradio as gr
+import imageio
+from yt_dlp import YoutubeDL
+import cv2
+import torch
+import torchvision
+import tempfile
+import numpy as np
+import smplx
+import pyrender
+import trimesh
+import trimesh.transformations as tra
+from dataclasses import dataclass
+from typing import List, Dict, Any
+import SkeletonDiffusion_Demo.plot_several_meshes as plot_several_meshes
+import SkeletonDiffusion_Demo.combine_video as combine
+import os
+
+# fix no display problem
+os.environ['PYOPENGL_PLATFORM'] = 'egl'
+os.system('export IMAGEMAGICK_BINARY=/home/stud/yaji/storage/user/yaji/NonisotropicSkeletonDiffusion/magick')
+
+
+# Load your torchscript model (ensure the path is correct)
+nlf_model = torch.jit.load("./models/nlf_l_multi.torchscript").cuda().eval()
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Load SMPL model (ensure the pkl file is correct)
+# This code assumes the SMPL_NEUTRAL.pkl model has its canonical orientation with the face along +X and up along +Y.
+smpl_model = smplx.create("./models/SMPL_NEUTRAL.pkl", model_type="smpl", gender="neutral").to(
+    device
+)
+smpl_params = []
+DESCRIPTION = "# SMPL Visualization Demo"
+FRAME_LIMIT = 100
+
+
+@dataclass
+class SMPLParams:
+    """
+    Data structure to hold SMPL parameters.
+    """
+
+    global_orient: torch.Tensor
+    body_pose: torch.Tensor
+    betas: torch.Tensor
+    transl: torch.Tensor
+
+
+def handle_video_input(video_file, youtube_url):
+    """Handles the video input: either a local file or a YouTube URL."""
+    if youtube_url:
+        ydl_opts = {
+            "format": "best",
+            "outtmpl": "downloads/%(title)s.%(ext)s",
+            "cookies": "cookies/cookies.txt",
+        }
+        with YoutubeDL(ydl_opts) as ydl:
+            info = ydl.extract_info(youtube_url, download=True)
+            video_path = ydl.prepare_filename(info)
+        return video_path
+    elif video_file:
+        return video_file
+    return None
+
+
+def correct_vertices(vertices):
+    """
+    Corrects the SMPL vertices to convert from the SMPL coordinate system to the renderer's coordinate system.
+
+    This version applies a rotation about the Y-axis by -90 degrees so that the original +X axis (assumed to be the
+    face direction) is transformed to the -Z axis (i.e., the model will face the camera if the camera is placed at
+    [0, 0, distance] looking along -Z). The up direction (Y axis) remains unchanged.
+    """
+    angle = np.radians(180)
+    # Build a 4x4 rotation matrix around Y
+    R = tra.rotation_matrix(angle, [1, 0, 0])
+    # Convert vertices to homogeneous coordinates (assumes vertices shape is (1, N, 3))
+    vertices_homo = np.hstack(
+        [vertices[0], np.ones((vertices[0].shape[0], 1))]
+    )  # shape: (N, 4)
+    vertices_corrected = (R @ vertices_homo.T).T  # Apply rotation
+    # Reshape back to (1, N, 3)
+    return vertices_corrected[:, :3].reshape(1, -1, 3)
+
+
+def render_smpl(vertices, width, height):
+    """
+    Renders the SMPL 3D model using PyRender.
+
+    - Applies a coordinate correction to the SMPL vertices.
+    - Builds a trimesh object and adds it to a pyrender scene.
+    - Sets up an orthographic camera with a given pose.
+    - Renders the scene offscreen.
+    - Converts the output image from RGB to BGR (to match OpenCV color format).
+    """
+    # Correct the vertices using the new rotation
+    vertices_corrected = correct_vertices(vertices)
+    # Create a trimesh mesh object using the corrected vertices and the SMPL faces
+    mesh = trimesh.Trimesh(vertices_corrected[0], smpl_model.faces)
+    scene = pyrender.Scene(
+        bg_color=[1.0, 1.0, 1.0, 0.9]
+    )  # Background color: white (RGB)
+    # remove background
+    mesh_node = pyrender.Mesh.from_trimesh(mesh)
+    scene.add(mesh_node)
+
+    # Set up an orthographic camera. Here we place the camera at [0, 0, distance] looking toward the origin.
+    camera = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0)
+    camera_pose = np.eye(4)
+    distance = 5.0
+    camera_pose[:3, 3] = [0, 0, distance]
+    scene.add(camera, pose=camera_pose)
+
+    # Render the scene using an offscreen renderer
+    renderer = pyrender.OffscreenRenderer(width, height)
+    color, _ = renderer.render(scene)
+    # Convert from RGB to BGR for OpenCV compatibility
+    color_bgr = cv2.cvtColor(color, cv2.COLOR_RGB2BGR)
+    return color_bgr
+
+
+def process_video(video_file, youtube_url):
+    """
+    Processes the input video and outputs a GIF:
+      - Obtains the video path.
+      - Reads frames from the video.
+      - Runs SMPL detection on each frame.
+      - Generates the SMPL mesh using the SMPL model.
+      - Renders the SMPL mesh.
+      - Blends the rendered SMPL visualization with the original frame.
+      - Saves the processed frames as a GIF.
+    """
+    input_path = handle_video_input(video_file, youtube_url)
+    if not input_path:
+        return None
+
+    output_path = tempfile.NamedTemporaryFile(suffix=".gif", delete=False).name
+
+    cap = cv2.VideoCapture(input_path)
+    frame_count = 0
+    smpl_params_list = []
+    rendered_smpl = None
+    frames = []
+    while cap.isOpened() and frame_count < FRAME_LIMIT:
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        image_tensor = torch.from_numpy(frame_rgb).permute(2, 0, 1).int().to(device)
+        with torch.inference_mode():
+            pred = nlf_model.detect_smpl_batched(image_tensor.unsqueeze(0))
+        pose_params = pred["pose"][0].cpu().numpy()
+        betas = pred["betas"][0].cpu().numpy()
+        transl = pred["trans"][0].cpu().numpy()
+
+        if pose_params.shape[0] == 0 and rendered_smpl is None:
+            print(f"No SMPL detected in frame {frame_count}")
+            frames.append(frame_rgb)
+            continue
+
+        if pose_params.shape[0] > 0:
+
+            smpl_param = SMPLParams(
+                global_orient=torch.tensor(pose_params[:, :3]).to(device),
+                body_pose=torch.tensor(pose_params[:, 3:]).to(device),
+                betas=torch.tensor(betas).to(device),
+                transl=torch.tensor(transl).to(device),
+            )
+            output_smpl = smpl_model(
+                global_orient=torch.tensor(pose_params[:, :3]).to(device),
+                body_pose=torch.tensor(pose_params[:, 3:]).to(device),
+                betas=torch.tensor(betas).to(device),
+                transl=torch.tensor(transl).to(device),
+            )
+
+            vertices = output_smpl.vertices.detach().cpu().numpy()
+            rendered_smpl = render_smpl(vertices, frame.shape[1], frame.shape[0])
+        smpl_params_list.append(smpl_param)
+        alpha = 0.6
+        blended = cv2.addWeighted(frame_rgb, 1 - alpha, rendered_smpl, alpha, 0)
+        frames.append(blended)
+        frame_count += 1
+
+    cap.release()
+
+    # Serialize SMPL parameters into a JSON-compatible format
+    smpl_params_serialized = [
+        {
+            "global_orient": p.global_orient.tolist(),
+            "body_pose": p.body_pose.tolist(),
+            "betas": p.betas.tolist(),
+            "transl": p.transl.tolist(),
+        }
+        for p in smpl_params_list
+    ]
+
+    # Save as GIF
+    imageio.mimsave(output_path, frames, fps=30, loop=0)
+    print(f"Output GIF saved to {output_path}")
+    return output_path, smpl_params_serialized
+
+
+def generate_motion_video(smpl_params_json: List[Dict[str, Any]]):
+    """
+    Generate a motion video from the given SMPL parameters.
+    """
+    # Deserialize JSON back into SMPLParams objects
+    smpl_params_list = [
+        SMPLParams(
+            global_orient=torch.tensor(p["global_orient"]),
+            body_pose=torch.tensor(p["body_pose"]),
+            betas=torch.tensor(p["betas"]),
+            transl=torch.tensor(p["transl"]),
+        )
+        for p in smpl_params_json
+    ]
+    # TODO: Using the SMPL parameters obtained from video, generate motion and save as .obj format, rank
+    # and find the closest to the ground truth and the farthest from the ground truth, just like the samples.
+
+    sample_obj_path = "./9622_GRAB/"
+    plot_several_meshes.main(sample_obj_path)
+    return combine.combine_video(sample_obj_path)
+
+
+with gr.Blocks(css="style.css") as demo:
+    gr.Markdown(DESCRIPTION)
+
+    with gr.Tabs():
+        with gr.Tab("Video Processing"):
+            with gr.Row():
+                with gr.Column():
+                    input_video = gr.Video(label="Input Video")
+                    youtube_url = gr.Textbox(label="YouTube URL")
+                    process_btn = gr.Button("Process Video")
+
+                with gr.Column():
+                    # output_video = gr.Video(label="SMPL Visualization")
+                    video_to_smpl = gr.Image(label="SMPL Visualization")
+                    # save smpl params in gradio
+                    obs_smpl_params = gr.JSON(label="SMPL Parameters")
+                    obs_smpl_params.visible = False
+                    generate_btn = gr.Button("Generate Motion")
+                    output_video = gr.Image(label="Generated Motion")
+
+            gr.Examples(
+                examples=sorted(pathlib.Path("downloads").glob("*.mp4")),
+                inputs=input_video,
+                outputs=video_to_smpl,
+                cache_examples=False,
+            )
+
+    process_btn.click(
+        fn=process_video,
+        inputs=[input_video, youtube_url],
+        outputs=[video_to_smpl, obs_smpl_params],
+    )
+    generate_btn.click(
+        fn=generate_motion_video, inputs=[obs_smpl_params], outputs=[output_video]
+    )
+
+demo.launch(server_name="0.0.0.0", share=True)

cookies/cookies.txt

@@ -0,0 +1,27 @@
+# Netscape HTTP Cookie File
+# http://curl.haxx.se/rfc/cookie_spec.html
+# This is a generated file!  Do not edit.
+
+.youtube.com	TRUE	/	FALSE	1771773809	HSID	AUsUjiw5uHpjnN9FQ
+.youtube.com	TRUE	/	TRUE	1771773809	SSID	ArWvlhFHWHJNe1ww7
+.youtube.com	TRUE	/	FALSE	1771773809	APISID	3V4Bd8r68qZSJoJR/Ag2KukueBOGGAojcc
+.youtube.com	TRUE	/	TRUE	1771773809	SAPISID	hjs3w0PI8zVyfzfa/A_B55o1Wk_drSalkA
+.youtube.com	TRUE	/	TRUE	1771773809	__Secure-1PAPISID	hjs3w0PI8zVyfzfa/A_B55o1Wk_drSalkA
+.youtube.com	TRUE	/	TRUE	1771773809	__Secure-3PAPISID	hjs3w0PI8zVyfzfa/A_B55o1Wk_drSalkA
+.youtube.com	TRUE	/	FALSE	1750438835	_ga	GA1.1.427766011.1715878835
+.youtube.com	TRUE	/	FALSE	1750588960	_ga_VCGEPY40VB	GS1.1.1716028960.2.0.1716028960.60.0.0
+.youtube.com	TRUE	/	TRUE	1751628628	LOGIN_INFO	AFmmF2swRQIhAKvoVorooDZFcdvOyhemewMTJIvSOBHPxRFnEibUwkQdAiAltzQ-cRmT4ZVM_K1G29I8kOyhqOcqwcs77DQc7aA0uA:QUQ3MjNmeGpEUmJFNXZNT2pRS2JqcnozdFMyN0xIc0pGUkNjaEd6ay01MXltZHlXQ0dpSkVpdG5rRUdRaWUxalZDLWN6RXYxZU5KMFphSE1URm9GT2ZJWHlfSDdCSHkzVzNsRGdCSFRndDZ1MGV4eEhvX1lBSHlQWG1Va04wellCaXFxOExPMkZUNUpMRS12NXUzVkFiTkFMbTJZTTluSm93
+.youtube.com	TRUE	/	TRUE	1742390134	NID	517=vTyhe8ky0TFImU6Zedz7cIBXkbO8_027L5Dlc2fcAN1FMVYtP401sld9WTCsRlLcQLC2tGyKwAEcygV4VZz_Z6fBCje4rAG_uJGP4BWyV_t_xTHeJzWJIdcSeziXRveABzB9CrEqSmQ9YNCVFWRFPsSohv5x06OTMSBaRrljZKDj8qfZl4xbF2fIDM08EiTK9u_UdIkeLvJjhJo7UnY6ot3St5z7W5UjAkw_aHoCfhvfL98tw3Op8g
+.youtube.com	TRUE	/	TRUE	1772581805	PREF	f7=4100&tz=Europe.Berlin&f4=4000000&f5=30000
+.youtube.com	TRUE	/	FALSE	1771773809	SID	g.a000sgjJU_L6MmDnLGVA_VkPxNDQBFRiuUjJSjiRMoSDGy1M4Q6ipIgBLsh1vRWZ01KcYNo0DQACgYKAZcSARQSFQHGX2MiEmIP-EgwP3AVZ9kO9dmvmhoVAUF8yKqgnQY68G5Egk_wIzyVPdIl0076
+.youtube.com	TRUE	/	TRUE	1771773809	__Secure-1PSID	g.a000sgjJU_L6MmDnLGVA_VkPxNDQBFRiuUjJSjiRMoSDGy1M4Q6iwBbPP-uFeHSg5_eYKOWU7AACgYKAa4SARQSFQHGX2MiyjrChB0tZfL__RkQAhA5PRoVAUF8yKrAHKtgrQDIty6zOxe2Zj-x0076
+.youtube.com	TRUE	/	TRUE	1771773809	__Secure-3PSID	g.a000sgjJU_L6MmDnLGVA_VkPxNDQBFRiuUjJSjiRMoSDGy1M4Q6iFLVYDARAAviR252WUqgzMAACgYKARwSARQSFQHGX2MiLFgk6z297qdWv19BK-t10xoVAUF8yKouCCN7Exq44094aUcZCFSm0076
+.youtube.com	TRUE	/	TRUE	1769557692	__Secure-1PSIDTS	sidts-CjEBmiPuTQiEzpQcYFOEjV6ORPmtcwo-bDKAiMIvMTgLUMhVblTUqg8srzKsE9GKZX2HEAA
+.youtube.com	TRUE	/	TRUE	1769557692	__Secure-3PSIDTS	sidts-CjEBmiPuTQiEzpQcYFOEjV6ORPmtcwo-bDKAiMIvMTgLUMhVblTUqg8srzKsE9GKZX2HEAA
+.youtube.com	TRUE	/	FALSE	1769557809	SIDCC	AKEyXzVOMGH2FCJRuD62uZGsy1IbhOWgrA-kH6qsslsuEJmKBEs-OIM6AKcylOWv9VvxRUIsQWU
+.youtube.com	TRUE	/	TRUE	1769557809	__Secure-1PSIDCC	AKEyXzUnJ-rqvN2XlLRlJ0t4Sd46J6u9BMuLD96kTuAoa15Ou6CEGGg3Zsbv5PubZpprUft-WRYf
+.youtube.com	TRUE	/	TRUE	1769557809	__Secure-3PSIDCC	AKEyXzVFIBk3DnxqCV-t9UcQRiQ-GI0XM8D_zSLmK_cPUYrQiTXU6iXMjc-uyaB2ipUpJ2NP9Hk
+.youtube.com	TRUE	/	TRUE	1753573803	VISITOR_INFO1_LIVE	VR8ZV-E7KNk
+.youtube.com	TRUE	/	TRUE	1753573803	VISITOR_PRIVACY_METADATA	CgJERRIEEgAgKRgB
+.youtube.com	TRUE	/	TRUE	0	YSC	GkwTh-BBCSY
+.youtube.com	TRUE	/	TRUE	1753571280	__Secure-ROLLOUT_TOKEN	CJfv8rXn-bvHAxCH75WF5euKAxi1w5yugpeLAw%3D%3D

gradio.sbatch

@@ -0,0 +1,16 @@
+#!/bin/bash
+#SBATCH --job-name="gradio_app_test"
+#SBATCH --cpus-per-task=4
+#SBATCH --gres=gpu:1,VRAM:24G
+#SBATCH --constraint="GPU_MODEL:rtx_6000"
+#SBATCH --mem=30G
+#SBATCH --time=96:00:00
+#SBATCH --mail-type=FAIL,INVALID_DEPEND,STAGE_OUT,TIME_LIMIT,REQUEUE
+#SBATCH --output=/home/stud/yaji/slurm/logs/slurm-%j.out
+#SBATCH --error=/home/stud/yaji/slurm/logs/slurm-%j.out
+
+GPU_TYPE=$(nvidia-smi -i 0 --query-gpu=name --format=csv,noheader)
+GPU_TYPE="${GPU_TYPE// /_}"
+echo $GPU_TYPE
+
+python app.py

magick

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:945eaba92f0d94fb4d54b45cfd67c2bdd163b987d66c6ca108d470b09049bfe0
+size 29795520

models/nlf_l_multi.torchscript

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2090b041ccebbd395018033152628c4f24afe519aed3a299861621547582cdad
+size 495696900

requirements.txt

@@ -0,0 +1,3 @@
+transformers
+torch
+yt_dlp

style.css

@@ -0,0 +1,11 @@
+h1 {
+  text-align: center;
+  display: block;
+}
+
+#duplicate-button {
+  margin: auto;
+  color: #fff;
+  background: #1565c0;
+  border-radius: 100vh;
+}

text.py

@@ -0,0 +1,8 @@
+import subprocess
+try:
+    subprocess.run(['ffmpeg', '-version'], check=True)
+except Exception as e:
+    print("FFmpeg not found:", e)
+print("Running FFmpeg command:", ' '.join(ffmpeg_cmd))
+print("FFmpeg output:", result.stdout)
+print("FFmpeg errors:", result.stderr)

tmp/requirements.txt

@@ -0,0 +1,2 @@
+transformers
+torch