diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..565d0bcb2930768808d9fa9e6f407e40d27aefbc 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,71 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +assets/demo_screenshot.png filter=lfs diff=lfs merge=lfs -text +assets/gmr_results.gif filter=lfs diff=lfs merge=lfs -text +assets/mujoco_result.gif filter=lfs diff=lfs merge=lfs -text +assets/protomotions_results.gif filter=lfs diff=lfs merge=lfs -text +assets/sonic_kimodo_demo.gif filter=lfs diff=lfs merge=lfs -text +assets/teaser.gif filter=lfs diff=lfs merge=lfs -text +docs/source/_static/arch.png filter=lfs diff=lfs merge=lfs -text +docs/source/_static/constraints.png filter=lfs diff=lfs merge=lfs -text +docs/source/_static/demo/editing_mode.png filter=lfs diff=lfs merge=lfs -text +docs/source/_static/demo/viewer.png filter=lfs diff=lfs merge=lfs -text +docs/source/_static/overview.png filter=lfs diff=lfs merge=lfs -text +docs/source/_static/skeletons/g1.png filter=lfs diff=lfs merge=lfs -text +docs/source/_static/skeletons/skeletons.png filter=lfs diff=lfs merge=lfs -text +docs/source/_static/skeletons/smplx.png filter=lfs diff=lfs merge=lfs -text +docs/source/_static/skeletons/soma.png filter=lfs diff=lfs merge=lfs -text +docs/source/_static/skeletons/soma_skels.png filter=lfs diff=lfs merge=lfs -text +docs/source/_static/smoothed_root.png filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/head_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_ankle_roll_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_index_0_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_index_1_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_middle_0_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_middle_1_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_palm_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_1_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_2_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_pitch_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_roll_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_yaw_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_knee_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_rubber_hand.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_pitch_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_roll_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_yaw_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_roll_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_roll_rubber_hand.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_yaw_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/logo_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/pelvis.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/pelvis_contour_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_ankle_roll_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_index_0_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_index_1_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_middle_0_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_middle_1_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_palm_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_1_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_2_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_pitch_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_roll_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_yaw_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_knee_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_rubber_hand.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_pitch_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_roll_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_yaw_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_roll_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_roll_rubber_hand.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_yaw_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_L_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_R_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link_23dof_rev_1_0.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link_rev_1_0.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/waist_constraint_L.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/waist_constraint_R.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/waist_support_link.STL filter=lfs diff=lfs merge=lfs -text +kimodo/assets/skeletons/g1skel34/meshes/g1/waist_yaw_link.STL filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..41a4d8632a4f66926cd956a98143e0b828f21a8e --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# debugging files +debug/ +SMPLX_NEUTRAL.npz + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +datasets + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# vscode +.vscode +*.code-workspace +/pyrightconfig.json +wandb/ + +# others +out +tmr_out +.ruff_cache +outputs +/debug +/batch*.sh +checkpoints/**/test/* +nohup.out + +*.swp +*.swo +*.txt~* +*.un~ +*~ +train_done +.aider* +onelogger.err + +# deploy files +/helm-library diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d4716f284b0ddbcb85badeb22c6eb9df01d0f30 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,50 @@ +repos: + # code formatting + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.4 + hooks: + - id: ruff + name: sort imports with ruff + args: [--select, I, --fix] + - id: ruff-format + name: format with ruff + + # docstring formatting + - repo: https://github.com/PyCQA/docformatter + rev: v1.7.7 + hooks: + - id: docformatter + args: + [ + --in-place, + --wrap-summaries=100, + --wrap-descriptions=100, + --style=sphinx, + ] + + # yaml formatting + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v3.0.0-alpha.6 + hooks: + - id: prettier + types: [yaml] + exclude: | + (?x)^( + environment\.yaml$ | + \.gitlab-ci\.yml$ | + \.k8s/.*\.(ya?ml)$ + ) + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace # Trims trailing whitespace. + - id: end-of-file-fixer # Makes sure files end in a newline and only a newline. + - id: check-yaml # Attempts to load all yaml files to verify syntax. + exclude: | + (?x)^( + \.gitlab-ci\.yml$ | + \.k8s/.*\.(ya?ml)$ + ) + +exclude: "checkpoints/.*" diff --git a/ATTRIBUTIONS.MD b/ATTRIBUTIONS.MD new file mode 100644 index 0000000000000000000000000000000000000000..ca10ff22ca65d441acfbceea3638894c69f7f909 --- /dev/null +++ b/ATTRIBUTIONS.MD @@ -0,0 +1,48 @@ +LLM2Vec MIT License https://github.com/McGill-NLP/llm2vec Copyright (c) 2024 McGill NLP + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Unitree mujoco BSD 3-Clause License https://github.com/unitreerobotics/unitree_mujoco/blob/main/LICENSE +Copyright (c) 2016-2024 HangZhou YuShu TECHNOLOGY CO.,LTD. ("Unitree Robotics") +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000000000000000000000000000000000..7313e855cdd615b38fe3e0479f58a22d7f5a3dd6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,84 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## [2026-05-03] + +### Fixed +- `benchmark/parse_folder.py` now averages each metric only over the testcases that actually report it. Previously, sparse constraint metrics (`constraint_root2d_acc`, `constraint_root2d_err`, `constraint_root2d_err_p95`, `constraint_fullbody_keyframe`, `constraint_end_effector`) were divided by the total motion count of the (split, category), including testcases of other constraint kinds that did not report the metric. This silently scaled values by `motions_with_metric / total_motions` (e.g. `constraint_root2d_acc` displayed as ~0.57 when the true value was ~0.93). Both the printed table and `summary_rows.json` are affected, including the combined constraints row that merges `constraints_withtext` and `constraints_notext`. Text-following metrics, foot-skate/contact metrics, and TMR metrics are unchanged. +- Updated Kimodo benchmark results in the documentation with this fix applied. + +## [2026-04-24] + +### Added +- Support for `TEXT_ENCODER_DEVICE` environment variable to force LLM2Vec to use the CPU instead of GPU. Setting `TEXT_ENCODER_DEVICE=cpu` reduces VRAM usage to <3 GB with a fairly small speed hit. +- `--save_example_dir` argument to `kimodo_gen` to save outputs to an example directory that can be directly loaded into `kimodo_demo` + +### Fixed +- Bug in post-processing that was incorrectly making the smoothed root the target for the root in full-body constraints +- Modified how transitions are handled in multi-prompt generation to improve smoothness + +### Removed +- `share_transition` and `percentage_transition_override` options from python API for multi-prompt generation + +## [2026-04-13] + +### Added +- Option `--bvh_standard_tpose` to use standard T-pose for BVH file saved from `generate.py` +- Option to use standard T-pose for BVH file saved or downloaded from demo +- Option to input/output BVH files that use standard T-pose with `motion_convert.py` +- Added BVH file containing the standard Kimodo T-pose to `kimodo/assets/skeletons/somaskel77/somaskel77_standard_tpose.bvh` +- Updated documentation with these new options + +## [2026-04-10] + +### Added +- [Kimodo-SOMA-RP-v1.1](https://huggingface.co/nvidia/Kimodo-SOMA-RP-v1.1) and [Kimodo-SOMA-SEED-v1.1](https://huggingface.co/nvidia/Kimodo-SOMA-SEED-v1.1) models and added support in the codebase. If not specified, the latest version of the models will be used automatically with the demo and CLI. +- [Kimodo Motion Generation Benchmark](https://huggingface.co/datasets/nvidia/Kimodo-Motion-Gen-Benchmark) for standardized evaluation of motion generation models training on the BONES-SEED dataset. +- Scripts to construct the full benchmark, generate motions for test cases, and compute evaluation metrics. +- Documentation explaining the benchmark and how to use the evaluation pipeline. +- [TMR-SOMA-RP-v1](https://huggingface.co/nvidia/TMR-SOMA-RP-v1) motion-text embedding model to be used for evaluation metrics. +- Added option to load LLM2Vec text encoder in fp32 precision. + +### Fixed +- Always use batch size 1 with LLM2Vec to avoid unexpected behavior of different embeddings based on batch size. +- Load LLM2Vec directly onto the GPU, if available. +- Updated documentation on constraints with more details. + +## [2026-04-01] + +### Fixed +- Fix unnecessary text encoder reload when switching between models in the interactive demo (if not using the text encoder server API). + +## [2026-03-31] + +### Added +- New `kimodo_convert` CLI tool for converting generated motions between formats (NPZ, BVH, MuJoCo CSV, AMASS NPZ). +- Support for loading and saving BVH, CSV, and NPZ motion files in the interactive demo. + +## [2026-03-27] + +### Fixed +- Bug fix for foot contact visualization in the interactive demo. +- Patch bug with BVH export for SOMA models. + +## [2026-03-19] + +### Changed +- **Breaking:** Model inputs/outputs now use the SOMA 77-joint skeleton (`somaskel77`). This affects saved motion formats and constraint files from previous versions. + +### Added +- Released timeline annotations for the BONES-SEED dataset on HuggingFace. + +## [2026-03-16] - Initial Release + +### Added +- Open-source release of Kimodo codebase under Apache-2.0 license. +- Five model variants: Kimodo-SOMA-RP-v1, Kimodo-G1-RP-v1, Kimodo-SOMA-SEED-v1, Kimodo-G1-SEED-v1, Kimodo-SMPLX-RP-v1. +- Command-line interface (`kimodo_gen`) for motion generation with text prompts and kinematic constraints. +- Interactive web-based motion authoring demo (`kimodo_demo`) with timeline editor, constraint tracks, and 3D visualization. +- Support for multiple output formats: default NPZ, MuJoCo qpos CSV (G1), AMASS NPZ (SMPL-X). +- Documentation site with quick start guide, installation instructions, CLI reference, and API docs. +- Compatibility with downstream tools: ProtoMotions (physics-based policy training) and GMR (motion retargeting). diff --git a/CONTRIBUTING.MD b/CONTRIBUTING.MD new file mode 100644 index 0000000000000000000000000000000000000000..35273ec12718a756c2756ffea799caa2f2b1673b --- /dev/null +++ b/CONTRIBUTING.MD @@ -0,0 +1,49 @@ +# How to Contribute + +## Code Reviews + +All submissions require review. We use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more information on using pull requests. + +## Signing Your Work + +* We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license. + + * Any contribution which contains commits that are not Signed-Off will not be accepted. + +* To sign off on a commit you simply use the `--signoff` (or `-s`) option when committing your changes: + ```bash + $ git commit -s -m "Add cool feature." + ``` + This will append the following to your commit message: + ``` + Signed-off-by: Your Name + ``` + +* Full text of the DCO: + + ``` + Developer Certificate of Origin + Version 1.1 + + Copyright (C) 2004, 2006 The Linux Foundation and its contributors. + 1 Letterman Drive + Suite D4700 + San Francisco, CA, 94129 + + Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + ``` + + ``` + Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or + + (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or + + (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. + + (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. + ``` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..077a4116c58f9d5692d63e3b4179a325ae1e827a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +FROM nvcr.io/nvidia/pytorch:24.10-py3 + +# Avoid some interactive prompts + make pip quieter/reproducible-ish +ENV DEBIAN_FRONTEND=noninteractive \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +# Where your code will live inside the container +WORKDIR /workspace + +# System deps +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl ca-certificates \ + cmake build-essential \ + gosu \ + && rm -rf /var/lib/apt/lists/* + +# Some base images ship a broken `/usr/local/bin/cmake` shim (from a partial pip install), +# which shadows `/usr/bin/cmake` and breaks builds that invoke `cmake` (e.g. MotionCorrection). +# Prefer the system cmake. +RUN rm -f /usr/local/bin/cmake || true + +# Install from docker_requirements.txt: kimodo editable (-e .), +# but MotionCorrection non-editable (./MotionCorrection). The -e . line ensures [project.scripts] +# from pyproject.toml are installed (kimodo_gen, kimodo_demo, kimodo_textencoder). +# SKIP_MOTION_CORRECTION_IN_SETUP=1 so setup.py does not bundle motion_correction; it is +# installed separately from ./MotionCorrection in the requirements file (non-editable). +COPY docker_requirements.txt /workspace/docker_requirements.txt +COPY setup.py /workspace/setup.py +COPY pyproject.toml /workspace/pyproject.toml +COPY kimodo /workspace/kimodo +COPY kimodo-viser /workspace/kimodo-viser +COPY MotionCorrection /workspace/MotionCorrection + +RUN --mount=type=cache,target=/root/.cache/pip \ + python -m pip install --upgrade pip \ + && SKIP_MOTION_CORRECTION_IN_SETUP=1 python -m pip install -r docker_requirements.txt + +# Use the docker-entrypoint script, to allow the docker to run as the actual user instead of root +COPY kimodo/scripts/docker-entrypoint.sh /usr/local/bin/docker-entrypoint +RUN chmod +x /usr/local/bin/docker-entrypoint + +# Default command (change to your entrypoint if you have one) +ENTRYPOINT ["docker-entrypoint"] +CMD ["bash"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..e048a61796b130714fac0dffe9a3c2a1faf9521d --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..2f75e520d25313cb4426662d907160bad9458d78 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include setup.py +recursive-include kimodo/assets * +recursive-include MotionCorrection/src *.cpp *.h *.inl +recursive-include MotionCorrection/python *.py *.dll +include MotionCorrection/CMakeLists.txt +include MotionCorrection/test_example.py diff --git a/MotionCorrection/.gitignore b/MotionCorrection/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..02744d122b35df207557c1c0a0332af84c7dce2f --- /dev/null +++ b/MotionCorrection/.gitignore @@ -0,0 +1,91 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +*.egg +*.egg-info/ +dist/ +build/ +*.whl +.Python +develop-eggs/ +.installed.cfg +pip-log.txt +pip-delete-this-directory.txt +.pytest_cache/ +.coverage +htmlcov/ +.tox/ +.venv +venv/ +ENV/ +env/ + +# C/C++ +*.o +*.obj +*.exe +*.out +*.app +*.dll +*.dylib +*.lib +*.a +*.la +*.lo +*.slo +*.ko +*.elf +*.ilk +*.map +*.exp +*.gch +*.pch +*.idb +*.pdb +*.mod +*.smod +*.lai + +# CMake +CMakeCache.txt +CMakeFiles/ +CMakeScripts/ +cmake_install.cmake +install_manifest.txt +CTestTestfile.cmake +_deps/ +cmake-build-*/ +CMakeUserPresets.json + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ +.DS_Store +*.iml +.project +.cproject +.settings/ + +# Visual Studio +.vs/ +*.user +*.suo +*.userosscache +*.sln.docstates +*.VC.db +*.VC.opendb + +# Build directories +build/ +Build/ +out/ +dist/ +temp/ + +# Logs +*.log diff --git a/MotionCorrection/CMakeLists.txt b/MotionCorrection/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0770898afe2c149d3cadc149f88ce38dc6756532 --- /dev/null +++ b/MotionCorrection/CMakeLists.txt @@ -0,0 +1,95 @@ +cmake_minimum_required(VERSION 3.15) +project(motion_correction) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# Find Python +find_package(Python3 COMPONENTS Interpreter Development REQUIRED) + +# Find or fetch pybind11 +find_package(pybind11 CONFIG QUIET) +if(NOT pybind11_FOUND) + message(STATUS "pybind11 not found, fetching from GitHub...") + include(FetchContent) + FetchContent_Declare( + pybind11 + GIT_REPOSITORY https://github.com/pybind/pybind11.git + GIT_TAG v2.11.1 + ) + FetchContent_MakeAvailable(pybind11) +endif() + +# Find or fetch Eigen +find_package(Eigen3 3.3 CONFIG QUIET) +if(NOT Eigen3_FOUND) + message(STATUS "Eigen3 not found, fetching from GitLab...") + include(FetchContent) + FetchContent_Declare( + Eigen + GIT_REPOSITORY https://gitlab.com/libeigen/eigen.git + GIT_TAG 3.4.0 + ) + set(EIGEN_BUILD_DOC OFF CACHE BOOL "" FORCE) + set(BUILD_TESTING OFF CACHE BOOL "" FORCE) + set(EIGEN_BUILD_PKGCONFIG OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(Eigen) +endif() + +# Source files +set(MATH_SOURCES + src/cpp/Math/Matrix.cpp + src/cpp/Math/Quaternion.cpp + src/cpp/Math/Transform.cpp + src/cpp/Math/Types.cpp + src/cpp/Math/Vector.cpp +) + +set(ANIM_SOURCES + src/cpp/AnimProcessing/InverseKinematics.cpp + src/cpp/AnimProcessing/TrajectoryCorrector.cpp + src/cpp/AnimProcessing/Utility.cpp +) + +# Create static library for the core functionality +add_library(motion_correction_cpp_base STATIC ${MATH_SOURCES} ${ANIM_SOURCES}) + +# Enable Position Independent Code (required for linking into shared library) +set_target_properties(motion_correction_cpp_base PROPERTIES POSITION_INDEPENDENT_CODE ON) + +target_include_directories(motion_correction_cpp_base PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/src/cpp +) + +if(TARGET Eigen3::Eigen) + target_link_libraries(motion_correction_cpp_base PUBLIC Eigen3::Eigen) +else() + target_link_libraries(motion_correction_cpp_base PUBLIC eigen) +endif() + +target_compile_definitions(motion_correction_cpp_base PUBLIC EIGEN_MPL2_ONLY) + +# Compiler-specific settings +if(MSVC) + # MSVC-specific flags + target_compile_options(motion_correction_cpp_base PRIVATE /W4 /arch:AVX) +else() + # GCC/Clang flags (also applies to MinGW on Windows) + # Enable SSE4.1 and AVX instructions for SIMD operations + target_compile_options(motion_correction_cpp_base PRIVATE -Wall -Wextra -msse4.1 -mavx) +endif() + +# Python bindings +pybind11_add_module(_motion_correction src/cpp/BindingsPython.cpp) + +target_link_libraries(_motion_correction PRIVATE motion_correction_cpp_base) + +target_include_directories(_motion_correction PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src/cpp +) + + +# Install the Python module +install(TARGETS _motion_correction LIBRARY DESTINATION python/motion_correction) +install(FILES python/motion_correction/__init__.py DESTINATION python/motion_correction) +install(FILES python/motion_correction/motion_postprocess.py DESTINATION python/motion_correction) diff --git a/MotionCorrection/MANIFEST.in b/MotionCorrection/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..4328751fd823876a0fbbd947b1de83a5aa6ccd78 --- /dev/null +++ b/MotionCorrection/MANIFEST.in @@ -0,0 +1,4 @@ +include CMakeLists.txt +include test_example.py +recursive-include src *.cpp *.h *.inl +recursive-include python *.py *.dll diff --git a/MotionCorrection/README.md b/MotionCorrection/README.md new file mode 100644 index 0000000000000000000000000000000000000000..051a1b4ee5ebc24ffe5d63f4a6c0d912a45d8125 --- /dev/null +++ b/MotionCorrection/README.md @@ -0,0 +1,31 @@ +# motion_correction + +Standalone `correct_motion` implementation packaged as a small C++ motion trajectory correction library with Python bindings. + +## Installation Guide + +### Prerequisites + +Ensure you have a C++17 compatible compiler (GCC 7.0+, Clang 5.0+, or MSVC 2017+) and CMake 3.15+. On Windows, install MinGW-w64 or Visual Studio with C++ tools. On Linux, install `build-essential` and `cmake`. + +This project will download and install additional third-party open source software projects. Review the license terms of these open source projects before use. + +### Build & Install + +#### Standard Installation +```bash +pip install . +``` + +#### Development Installation +```bash +pip install -e . +``` + +### Verify Installation + +```python +import motion_correction +print("Installation successful!") +``` +You can also run `python run_test.py` for a simple test. diff --git a/MotionCorrection/python/motion_correction/__init__.py b/MotionCorrection/python/motion_correction/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..571d705108db0bba53f667a193ccb5a15ab69d95 --- /dev/null +++ b/MotionCorrection/python/motion_correction/__init__.py @@ -0,0 +1 @@ +from ._motion_correction import * diff --git a/MotionCorrection/python/motion_correction/motion_postprocess.py b/MotionCorrection/python/motion_correction/motion_postprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..eb6270a5091a2fc214e977e132dd05065bae9640 --- /dev/null +++ b/MotionCorrection/python/motion_correction/motion_postprocess.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import pickle + +import numpy as np +import torch + +import motion_correction + + +def correct_motion( + hipTranslations, + jointRotations, + contacts, + hipTranslationsInput, + rotationsInput, + constraint_masks, + contact_threshold, + root_margin, + working_rig, + has_double_ankle_joints=False, +): + joint_names = [x.name for x in working_rig] + joint_parents = [ + joint_names.index(working_rig[i].parent) if working_rig[i].parent in joint_names else -1 + for i in range(len(working_rig)) + ] + joint_ref_translations = [list(x.t_pose_translation) for x in working_rig] + joint_ref_rotations = [list(x.t_pose_rotation) for x in working_rig] + + left_hand_idx = [i for i in range(len(joint_names)) if working_rig[i].retarget_tag == "LeftHand"] + if len(left_hand_idx) != 1: + raise RuntimeError(f"correct_motion: Expected exactly one joint with LeftHand tag") + left_hand_idx = left_hand_idx[0] + + right_hand_idx = [i for i in range(len(joint_names)) if working_rig[i].retarget_tag == "RightHand"] + if len(right_hand_idx) != 1: + raise RuntimeError(f"correct_motion: Expected exactly one joint with RightHand tag") + right_hand_idx = right_hand_idx[0] + + left_foot_idx = [i for i in range(len(joint_names)) if working_rig[i].retarget_tag == "LeftFoot"] + if len(left_foot_idx) != 1: + raise RuntimeError(f"correct_motion: Expected exactly one joint with LeftFoot tag") + left_foot_idx = left_foot_idx[0] + + right_foot_idx = [i for i in range(len(joint_names)) if working_rig[i].retarget_tag == "RightFoot"] + if len(right_foot_idx) != 1: + raise RuntimeError(f"correct_motion: Expected exactly one joint with RightFoot tag") + right_foot_idx = right_foot_idx[0] + + end_frame = hipTranslations.shape[1] + + default_mask = torch.zeros(hipTranslations.shape[1], dtype=torch.float32) + root_mask = constraint_masks.get("Root", default_mask) + full_body_mask = constraint_masks.get("FullBody", default_mask) + left_hand_mask = constraint_masks.get("LeftHand", default_mask) + right_hand_mask = constraint_masks.get("RightHand", default_mask) + left_foot_mask = constraint_masks.get("LeftFoot", default_mask) + right_foot_mask = constraint_masks.get("RightFoot", default_mask) + + batch_size = hipTranslations.shape[0] + + for b in range(batch_size): + hipTranslationsCorrected = hipTranslations[b, :end_frame].detach().cpu().flatten().numpy().astype(np.float32) + rotationsCorrected = jointRotations[b, :end_frame].detach().cpu().flatten().numpy().astype(np.float32) + + hipTranslationsInput_flat = hipTranslationsInput.detach().cpu().flatten().numpy().astype(np.float32) + rotationsInput_flat = rotationsInput.detach().cpu().flatten().numpy().astype(np.float32) + ctcs = contacts[b].detach().cpu().flatten().numpy().astype(np.float32) + + motion_correction.correct_motion( + hipTranslationsCorrected, + rotationsCorrected, + hipTranslationsInput_flat, + rotationsInput_flat, + full_body_mask, + left_hand_mask, + right_hand_mask, + left_foot_mask, + right_foot_mask, + root_mask, + np.array(ctcs, dtype=np.float32), + joint_parents, + joint_ref_translations, + joint_ref_rotations, + left_hand_idx, + right_hand_idx, + left_foot_idx, + right_foot_idx, + contact_threshold, + root_margin, + has_double_ankle_joints, + ) + + hipTranslations[b, :end_frame] = torch.from_numpy( + hipTranslationsCorrected.reshape(*hipTranslations[b, :end_frame].shape) + ) + jointRotations[b, :end_frame] = torch.from_numpy( + rotationsCorrected.reshape(*jointRotations[b, :end_frame].shape) + ) diff --git a/MotionCorrection/run_test.py b/MotionCorrection/run_test.py new file mode 100644 index 0000000000000000000000000000000000000000..1e1032602b923b1e5c48c0971c3ee1bb91ffa7b2 --- /dev/null +++ b/MotionCorrection/run_test.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import torch +from motion_correction.motion_postprocess import correct_motion + + +class Joint: + def __init__(self, name, parent, t_pose_translation, t_pose_rotation, retarget_tag=""): + self.name = name + self.parent = parent + self.t_pose_translation = t_pose_translation + self.t_pose_rotation = t_pose_rotation + self.retarget_tag = retarget_tag + + +def create_test_rig(): + return [ + Joint("Hips", None, [0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0], "Root"), + Joint("Spine", "Hips", [0.0, 0.1, 0.0], [0.0, 0.0, 0.0, 1.0]), + Joint("LeftUpLeg", "Hips", [-0.1, -0.05, 0.0], [0.0, 0.0, 0.0, 1.0]), + Joint("LeftLeg", "LeftUpLeg", [0.0, -0.4, 0.0], [0.0, 0.0, 0.0, 1.0]), + Joint("LeftFoot", "LeftLeg", [0.0, -0.4, 0.0], [0.0, 0.0, 0.0, 1.0], "LeftFoot"), + Joint("RightUpLeg", "Hips", [0.1, -0.05, 0.0], [0.0, 0.0, 0.0, 1.0]), + Joint("RightLeg", "RightUpLeg", [0.0, -0.4, 0.0], [0.0, 0.0, 0.0, 1.0]), + Joint("RightFoot", "RightLeg", [0.0, -0.4, 0.0], [0.0, 0.0, 0.0, 1.0], "RightFoot"), + Joint("LeftArm", "Spine", [-0.3, 0.3, 0.0], [0.0, 0.0, 0.0, 1.0]), + Joint("LeftHand", "LeftArm", [-0.3, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0], "LeftHand"), + Joint("RightArm", "Spine", [0.3, 0.3, 0.0], [0.0, 0.0, 0.0, 1.0]), + Joint("RightHand", "RightArm", [0.3, 0.0, 0.0], [0.0, 0.0, 0.0, 1.0], "RightHand"), + ] + + +if __name__ == "__main__": + # Test data + batch_size, num_frames, num_joints = 1, 60, 12 + + hipTranslations = torch.randn(batch_size, num_frames, 3) + jointRotations = torch.randn(batch_size, num_frames, num_joints, 4) + jointRotations = jointRotations / jointRotations.norm(dim=-1, keepdim=True) + + contacts = torch.rand(batch_size, num_frames, 4) + hipTranslationsInput = hipTranslations.clone() + rotationsInput = jointRotations.clone() + + constraint_masks = { + "Root": torch.zeros(num_frames), + "FullBody": torch.zeros(num_frames), + "LeftHand": torch.zeros(num_frames), + "RightHand": torch.zeros(num_frames), + "LeftFoot": torch.zeros(num_frames), + "RightFoot": torch.zeros(num_frames), + } + + working_rig = create_test_rig() + + # Run correction + correct_motion( + hipTranslations=hipTranslations, + jointRotations=jointRotations, + contacts=contacts, + hipTranslationsInput=hipTranslationsInput, + rotationsInput=rotationsInput, + constraint_masks=constraint_masks, + contact_threshold=0.5, + root_margin=0.01, + working_rig=working_rig, + ) + + print("Test completed successfully") diff --git a/MotionCorrection/setup.py b/MotionCorrection/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..8d450ec11664015f106ac2e640d0f29aee3702c8 --- /dev/null +++ b/MotionCorrection/setup.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Setup script for correct_motion standalone package.""" + +import os +import shutil +import subprocess +import sys +from pathlib import Path + +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext + + +class CMakeExtension(Extension): + def __init__(self, name, sourcedir=""): + Extension.__init__(self, name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + + +class CMakeBuild(build_ext): + def run(self): + try: + subprocess.check_output(["cmake", "--version"]) + except OSError: + raise RuntimeError("CMake must be installed to build this package") + + for ext in self.extensions: + self.build_extension(ext) + + def build_extension(self, ext): + # import pdb; pdb.set_trace() # Debug build process + + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + cmake_args = [ + f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}", + f"-DPYTHON_EXECUTABLE={sys.executable}", + ] + + cfg = "Debug" if self.debug else "Release" + build_args = ["--config", cfg] + + cmake_args += [f"-DCMAKE_BUILD_TYPE={cfg}"] + + use_mingw = False + mingw_bin = None + + if sys.platform == "win32": + generator = os.environ.get("CMAKE_GENERATOR", "") + if generator: + cmake_args = ["-G", generator] + cmake_args + if "mingw" in generator.lower(): + use_mingw = True + else: + cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] + else: + # Try MinGW Makefiles as default on Windows + try: + subprocess.check_output(["g++", "--version"], stderr=subprocess.STDOUT) + use_mingw = True + cmake_args = ["-G", "MinGW Makefiles"] + cmake_args + build_args = [] # MinGW Makefiles do not accept --config + except (OSError, subprocess.CalledProcessError): + # If g++ is not found, let CMake use its default (Visual Studio) + cmake_args += [f"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{cfg.upper()}={extdir}"] + + if use_mingw: + gxx_path = shutil.which("g++") + if gxx_path: + mingw_bin = Path(gxx_path).parent + else: + build_args += ["--", "-j4"] + + env = os.environ.copy() + env["CXXFLAGS"] = f'{env.get("CXXFLAGS", "")} -DVERSION_INFO=\\"{self.distribution.get_version()}\\"' + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + + subprocess.check_call(["cmake", ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env) + subprocess.check_call(["cmake", "--build", "."] + build_args, cwd=self.build_temp) + + if use_mingw and mingw_bin is not None: + runtime_libs = [ + "libstdc++-6.dll", + "libgcc_s_seh-1.dll", + "libwinpthread-1.dll", + ] + extdir_path = Path(extdir) + extdir_path.mkdir(parents=True, exist_ok=True) + for lib_name in runtime_libs: + src_path = mingw_bin / lib_name + if src_path.exists(): + shutil.copy2(src_path, extdir_path / lib_name) + else: + self.announce( + f"Warning: Expected MinGW runtime DLL '{lib_name}' not found next to g++ (looked in {mingw_bin}). " + "The built extension may fail to import if the DLL is not on PATH.", + level=3, + ) + + +setup( + name="motion_correction", + version="1.0.0", + author="NVIDIA", + description="Standalone correct_motion function", + long_description="", + packages=["motion_correction"], + package_dir={"": "python"}, + ext_modules=[CMakeExtension("motion_correction._motion_correction")], + cmdclass={"build_ext": CMakeBuild}, + zip_safe=False, + python_requires=">=3.8", + install_requires=[ + "torch>=1.10.0", + "numpy>=1.19.0", + # 'cmake' # can install this via pip if the windows system does not have it. But need to run this by yourself before build, not in here. + ], +) diff --git a/MotionCorrection/src/cpp/AnimProcessing/InverseKinematics.cpp b/MotionCorrection/src/cpp/AnimProcessing/InverseKinematics.cpp new file mode 100644 index 0000000000000000000000000000000000000000..21eb186b8f099281e2d764f27f39636444857c8d --- /dev/null +++ b/MotionCorrection/src/cpp/AnimProcessing/InverseKinematics.cpp @@ -0,0 +1,200 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "InverseKinematics.h" +#include "Math/Scalar.h" +#include + + +using namespace IK; + +namespace +{ + +float getAngleWithTwoSideVectors(const Math::Vector& vecLeft, const Math::Vector& vecRight) +{ + auto lNorm = vecLeft.GetNormalized3(); + auto rNorm = vecRight.GetNormalized3(); + + float cosine = lNorm.GetDot3(rNorm); + float sine = lNorm.Cross3(rNorm).GetLength3(); + + return atan2f(sine, cosine); // in radian +} + +float getAngleWithCosineRule (const float lSideLeft, const float lSideRight, const float lSideAcross) +{ + float val = + (lSideRight * lSideRight + lSideLeft * lSideLeft - lSideAcross * lSideAcross) / + (2.0f * lSideLeft * lSideRight); + val = Math::Clamp(val, -1.0f, 1.0f); // numerical stability. also avoid impossible trangulars + return acosf(val); // in radian +} + +} + + +void IK::TwoBoneIk( + Pose& pose, + const Math::Transform& rootTransform, + uint32_t cIdx, + float weight, + const Math::Vector& target, + const std::vector& joint_parents_vec, + const Math::Vector& hintOffset +) +{ + weight = Math::Clamp(weight, 0.0f, 1.0f); + if (!(weight > 0.0f)) + return; + + // Two bone IK: joints are represented as "a", "b", "c" in the below comments: + // 1. stage 1, bend joint a and joint b, so that |ac| = |at|, while vec_ac maintain the same direction + // 2. stage 2, rotate start joint a so that c and t are in the same place + + // a a a | + // |\ |\ |\ | + // | \ | \ | \ | + // | \ (stage 1 ->) | \ (stage 2 ->) | \ | + // | b | b | b | + // | \ | | | / | + // | \ | | | / | + // t c t c t(c) | + // (a is the root joint, b is the middle joint and c is the end joint) + // + + int32_t bIdx = joint_parents_vec[cIdx]; + if (bIdx < 0) + { + return; + } + int32_t aIdx = joint_parents_vec[bIdx]; + if (aIdx < 0) + { + return; + } + + // Find the parent world transform of joint a: + Math::Transform aParentWorldTransform = Math::Transform::Identity; + int32_t idx = joint_parents_vec[aIdx]; + while (idx >= 0) + { + aParentWorldTransform = aParentWorldTransform * pose[idx]; + idx = joint_parents_vec[idx]; + } + aParentWorldTransform = aParentWorldTransform * rootTransform; + + // Compute world space transforms of a, b and c: + Math::Transform aWorld = pose[aIdx] * aParentWorldTransform; + Math::Transform bWorld = pose[bIdx] * aWorld; + Math::Transform cWorld = pose[cIdx] * bWorld; + + auto a = aWorld.GetTranslation(); + auto b = bWorld.GetTranslation(); + auto c = cWorld.GetTranslation(); + auto t = Math::Vector::Lerp(c, target, weight); + + // step 1 (stage 1): extend / contract the joint chain to match the distance + float eps = 0.0001f; // numerical stability + float l_ab = (b - a).Length3().GetX(); + float l_bc = (c - b).Length3().GetX(); + float l_at = (a - t).Length3().GetX(); + l_at = Math::Clamp(l_at, eps, (l_ab + l_bc) * 0.999f); // when not reachable, replace with maximum reachable length + + // get the current angles + float theta_bac_current = getAngleWithTwoSideVectors(a - b, a - c); + float theta_abc_current = getAngleWithTwoSideVectors(b - a, b - c); + // get the desired angles + if (l_ab < eps || l_bc < eps || l_at < eps) + { + return; // the length is too small. rejecting potentially numerically unstable requests. + } + float theta_bac_desired = getAngleWithCosineRule(l_ab, l_at, l_bc); + float theta_abc_desired = getAngleWithCosineRule(l_ab, l_bc, l_at); + + // in joint[0]'s parent's space + Math::Vector rotationAxis = Math::Vector::Cross3(c - a, bWorld.TransformPoint(hintOffset) - a); + float l = rotationAxis.GetLength3(); + if (l == 0) + { + rotationAxis = Math::Vector(0,0,1); + } + else + { + rotationAxis /= l; + } + + // get the rotation with axis in the local space of joint a and joint b + Math::Vector rotationAxisLocalInBSpace = bWorld.GetRotation().RotateVectorInverse(rotationAxis); + Math::Transform rotateInB( + Math::Quaternion(rotationAxisLocalInBSpace, + (theta_abc_desired - theta_abc_current)), Math::Vector::Zero); + + pose[bIdx] = rotateInB * pose[bIdx]; + + Math::Vector rotationAxisLocalInASpace = aWorld.GetRotation().RotateVectorInverse(rotationAxis); + Math::Transform rotateInA( + Math::Quaternion(rotationAxisLocalInASpace, + (theta_bac_desired - theta_bac_current)), Math::Vector::Zero); + + pose[aIdx] = rotateInA * pose[aIdx]; + + // recompute a's world space transform as we're going to need it: + aWorld = pose[aIdx] * aParentWorldTransform; + + // step 2 (stage 2): rotate joint a so that the target and the end joint c matches + auto acLocal = aWorld.GetRotation().RotateVectorInverse( + c - a); + auto atLocal = aWorld.GetRotation().RotateVectorInverse( + target - a); + Math::Transform rotateStageTwo( + Math::Quaternion::FromRotationBetweenVectors(acLocal, atLocal), Math::Vector::Zero + ); + + pose[aIdx] = rotateStageTwo * pose[aIdx]; + +} + +void IK::OneBoneIk( + Pose& pose, + const Math::Transform& rootTransform, + uint32_t bIdx, + float weight, + const Math::Vector& target, + const std::vector& joint_parents_vec +) +{ + weight = Math::Clamp(weight, 0.0f, 1.0f); + if (!(weight > 0.0f)) + return; + + int32_t aIdx = joint_parents_vec[bIdx]; + if (aIdx < 0) + { + return; + } + + // Find the parent world transform of joint a: + Math::Transform aParentWorldTransform = Math::Transform::Identity; + int32_t idx = joint_parents_vec[aIdx]; + while (idx >= 0) + { + aParentWorldTransform = aParentWorldTransform * pose[idx]; + idx = joint_parents_vec[idx]; + } + aParentWorldTransform = aParentWorldTransform * rootTransform; + + // Compute world space transforms of a, b and c: + Math::Transform aWorld = pose[aIdx] * aParentWorldTransform; + Math::Transform bWorld = pose[bIdx] * aWorld; + + auto abLocal = aWorld.GetRotation().RotateVectorInverse( + bWorld.GetTranslation() - aWorld.GetTranslation()); + auto atLocal = aWorld.GetRotation().RotateVectorInverse( + target - aWorld.GetTranslation()); + + auto deltaRLocal = Math::Quaternion::NLerp(Math::Quaternion::Identity, Math::Quaternion::FromRotationBetweenVectors(abLocal, atLocal), weight); + pose[aIdx].SetRotation(deltaRLocal * pose[aIdx].GetRotation()); +} diff --git a/MotionCorrection/src/cpp/AnimProcessing/InverseKinematics.h b/MotionCorrection/src/cpp/AnimProcessing/InverseKinematics.h new file mode 100644 index 0000000000000000000000000000000000000000..bf70b4815faebf3afea33310b3cd07047eb55be5 --- /dev/null +++ b/MotionCorrection/src/cpp/AnimProcessing/InverseKinematics.h @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Math/Transform.h" + +#include + +using Pose = std::vector; + +namespace IK { + + void TwoBoneIk( + Pose& pose, + const Math::Transform& rootTransform, + uint32_t jointIdx, + float weight, + const Math::Vector& target, + const std::vector& joint_parents_vec, + const Math::Vector& hintOffset = Math::Vector::Zero + ); + + void OneBoneIk( + Pose& pose, + const Math::Transform& rootTransform, + uint32_t jointIdx, + float weight, + const Math::Vector& target, + const std::vector& joint_parents_vec + ); + +} diff --git a/MotionCorrection/src/cpp/AnimProcessing/TrajectoryCorrector.cpp b/MotionCorrection/src/cpp/AnimProcessing/TrajectoryCorrector.cpp new file mode 100644 index 0000000000000000000000000000000000000000..eaf8799de7cb896ca4c9ad3c083d0d5d0e673327 --- /dev/null +++ b/MotionCorrection/src/cpp/AnimProcessing/TrajectoryCorrector.cpp @@ -0,0 +1,524 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "TrajectoryCorrector.h" +#include + +static void removeRows( + Eigen::SparseMatrix& M, + Eigen::MatrixXd *v, + int minCoeffs) +{ + Eigen::SparseMatrix rowMajorMat = M; + rowMajorMat.makeCompressed(); // Ensure compressed format + + std::vector> triplets; + triplets.reserve(rowMajorMat.nonZeros()); + + int newRow = 0; + for (int i = 0; i < rowMajorMat.outerSize(); ++i) { + // Get nonzero count via outerIndexPtr (compressed format only) + int nnz = rowMajorMat.outerIndexPtr()[i + 1] - rowMajorMat.outerIndexPtr()[i]; + + if (nnz >= minCoeffs) { + // Iterate through nonzeros in this row + for (Eigen::SparseMatrix::InnerIterator it(rowMajorMat, i); it; ++it) { + triplets.emplace_back(newRow, it.col(), it.value()); + } + if (v) + { + v->row(newRow) = v->row(i); + } + newRow++; + } + } + + M = Eigen::SparseMatrix(newRow, M.cols()); + M.setFromTriplets(triplets.begin(), triplets.end()); + if (v) + { + v->conservativeResize(newRow, v->cols()); + } +} + +static void multVelWeights( + Eigen::SparseMatrix& V, + Eigen::MatrixXd* v_rhs, + const Eigen::VectorXd& velocityWeights +) +{ + Eigen::SparseMatrix rowMajorMat = V; + rowMajorMat.makeCompressed(); // Ensure compressed format + + std::vector> triplets; + triplets.reserve(rowMajorMat.nonZeros()); + + for (int i = 0; i < rowMajorMat.outerSize(); ++i) { + // Iterate through nonzeros in this row + Eigen::SparseMatrix::InnerIterator it(rowMajorMat, i); + double vel_weight = velocityWeights[it.col()]; + for(; it; ++it) + { + triplets.emplace_back(i, it.col(), it.value() * vel_weight); + } + if (v_rhs) + { + (*v_rhs).row(i) = (*v_rhs).row(i) * vel_weight; + } + } +} + + +void TrajectoryCorrector::computeDiffMats( + Eigen::SparseMatrix& V, + Eigen::SparseMatrix& A, + uint32_t N, + const Eigen::VectorXd& velocityWeights, + Eigen::MatrixXd* v_rhs, + Eigen::MatrixXd* a_rhs) +{ + std::vector> tripletList; + + // Identity matrix" + tripletList.clear(); + Eigen::SparseMatrix I(N, N); + for (uint32_t i = 0; i < N; ++i) + { + tripletList.emplace_back(i, i, 1); + } + I.setFromTriplets(tripletList.begin(), tripletList.end()); + + // urr, a time translation operator? Gives you the value on the next frame. + // Leave the last row blank because that's the end of the timeline. + tripletList.clear(); + Eigen::SparseMatrix T(N, N); + Eigen::MatrixXd t_rhs; + for(uint32_t i = 0; i < N-1; ++i) + { + // next frame is + tripletList.emplace_back(i, i+1, 1.0); + } + T.setFromTriplets(tripletList.begin(), tripletList.end()); + + // v = Tx + t_rhs - x; + // v = (T - I)x + t_rhs; + V = T - I; + if (v_rhs) + { + *v_rhs = t_rhs; + } + removeRows(V, v_rhs, 2); + + // a = -x + 2 (T x + t_rhs) - (T (T x + t_rhs) + t_rhs) + // a = (-I + 2 T - T^2) x + t_rhs - T t_rhs + A = 2 * T - I - T * T; + if (a_rhs) + { + *a_rhs = t_rhs - T * t_rhs; + } + removeRows(A, a_rhs, 3); + + if (velocityWeights.size() > 0) + { + multVelWeights(V, v_rhs, velocityWeights); + } +} + +TrajectoryCorrector::TrajectoryCorrector( + const Eigen::VectorXd& margins, + float pos_weight, + float vel_weight, + float acc_weight, + const Eigen::VectorXd& velocityWeights, + uint32_t admm_iters ) : + m_admm_iters(admm_iters) +{ + + // This class is used to modify a trajectory to hit specific values at + // specific frames, while respecting the following soft constraints: + + // * Preserve the original positions + // * Preserve the original velocities + // * Preserve the original accelerations + + // The weights of these soft constraints are specified in "pos_weight" etc. + + // This is posed as a minimization problem: + + // E(x) = pos_weight * |x - x_orig|^2 + vel_weight * |V x - V x_orig| + acc_weight * |A x - A x_orig| + + // where you minimize E(x) subject to specified values at indices where "mask" + // is equal to 1. V is a matrix that computes the N-1 velocities between frame n-1 and frame n, + // and A computes the N-2 accelerations associated with frames n-1, n and n+1. + + // In addition to this, there are constraints where the trajectory is allowed to + // deviate from the target points by a maximum margin. The "margins" input to this + // constructor specifies what type of constraint is active on a particular frame: + + // margins[0] < 0 ==> unconstrained + // margins[i] == 0 ==> pinned on this frame + // margins[i] > 0 ==> can deviate within the margin + + // I'm solving the optimization problem using ADMM, ie following equations + // 8,9,10 on this paper: + + // https://mattoverby.net/files/admm-pd-overby17.pdf + + uint32_t N = uint32_t(margins.rows()); + for(uint32_t i = 0; i < N; ++i) + { + if( margins[i] > 0 ) + { + m_margin_locs.push_back(i); + m_margin_vals.push_back(margins[i]); + } + + if(margins[i] == 0) + { + m_constrained_locs.push_back(i); + } + else + { + m_unconstrained_locs.push_back(i); + } + } + + Eigen::SparseMatrix V, A; + computeDiffMats( + V, A, + N, velocityWeights + ); + + // build an identity matrix: + std::vector> tripletList; + Eigen::SparseMatrix I(N, N); + for (uint32_t i = 0; i < N; ++i) + { + tripletList.emplace_back(i, i, 1.0f); + } + I.setFromTriplets(tripletList.begin(), tripletList.end()); + + /* + self.N = ( + self.pos_weight * torch.diag_embed(torch.full_like(interp_mask, 1)) + + self.vel_weight * torch.matmul(self.V.T, self.V) + + self.acc_weight * torch.matmul(self.A.T, self.A) + ) + */ + + m_N = pos_weight * I + vel_weight * (V.transpose() * V) + acc_weight * (A.transpose() * A); + + double diagMax = 0; + for (uint32_t i = 0; i < N; ++i) + { + diagMax = std::max(m_N.coeff(i,i), diagMax); + } + m_admm_stepsize = 0.5f * sqrtf(float(diagMax)); + + /* + M = ( + self.N + + self.admm_stepsize * torch.matmul(self.S.T, self.S) + ) + */ + tripletList.clear(); + Eigen::SparseMatrix M(N, N); + for( auto i : m_margin_locs) + { + tripletList.emplace_back(i, i, m_admm_stepsize); + } + M.setFromTriplets(tripletList.begin(), tripletList.end()); + M += m_N; + + /* + self.lhsmat = torch.matmul(self.U.T, torch.matmul(self.M, self.U)) + self.lhsmat_inv = torch.inverse(self.lhsmat) + */ + tripletList.clear(); + Eigen::SparseMatrix S(m_unconstrained_locs.size(), N); + for (uint32_t i = 0; i < m_unconstrained_locs.size(); ++i) + { + uint32_t ifull = m_unconstrained_locs[i]; + tripletList.emplace_back(i, ifull, 1.0f); + } + S.setFromTriplets(tripletList.begin(), tripletList.end()); + M = S * M * S.transpose(); + + if(m_unconstrained_locs.size()) + { + m_system_lu.compute(M); + } +} + + +void TrajectoryCorrector::Interpolate( + Eigen::MatrixXd& x, + const Eigen::MatrixXd& observations, + const Eigen::MatrixXd& ref_positions +) const +{ + if( + m_constrained_locs.empty() && + m_margin_locs.empty() + ) + { + x = ref_positions; + return; + } + + uint32_t numCols = uint32_t(x.cols()); + if(m_margin_locs.empty()) + { + x_update( + x, + Eigen::MatrixXd(0, numCols), + Eigen::MatrixXd(0, numCols), + ref_positions, + observations + ); + } + else + { + x = ref_positions; + Eigen::MatrixXd z(m_margin_locs.size(), numCols); + Eigen::MatrixXd z_t(m_margin_locs.size(), numCols); + Eigen::MatrixXd u(m_margin_locs.size(), numCols); + for( uint32_t i = 0; i < m_margin_locs.size(); ++i) + { + for(uint32_t j = 0; j < numCols; ++j) + { + z_t(i, j) = observations(m_margin_locs[i], j); + z(i, j) = ref_positions(m_margin_locs[i], j); + u(i, j) =0; + } + } + + for(uint32_t i = 0; i < m_admm_iters; ++i) + { + x_update( + x, + z, + u, + ref_positions, + observations + ); + z_update(z, x, z_t, u); + u_update(u, x, z); + } + } + +} + +void TrajectoryCorrector::x_update( + Eigen::MatrixXd &x, + const Eigen::MatrixXd &z, + const Eigen::MatrixXd &u, + const Eigen::MatrixXd &x_t, // reference positions - defines the original shape of the curve that we want to preserve + const Eigen::MatrixXd &x_o // target positions for constraints +) const +{ + + uint32_t numRows = uint32_t(x.rows()); + uint32_t numCols = uint32_t(x.cols()); + + // Here's what we're minimizing with ADMM: + // min f(x) + g(z) + // s.t A x + B z = c + + // Make these choices so that z = S x: + // A = S, B = -I, c = 0 + // + // g(z) = infinity when it's too far away from z_target, zero otherwise + // + // f(x) penalizes deviations in position, velocity and acceleration + // from a reference trajectory: + // + // f(x) = 1/2( + // kx |I x - x_t|^2 + + // kv |V x - v_t|^2 + + // kx |A x - a_t|^2 + // ) + // + // It's also infinite when components of x devaiate from their target + // values where they're pinned... + + // Substituting the matrices into the standard admm update rules gives us this: + // x{n+1} = argmin(f(x) + ρ/2 |S x - z{n} + u{n}|^2) + // z{n+1} = argmin(g(z) + ρ/2 |S x{n+1} - z + u{n}|^2) + // u{n+1} = u{n} + (S x{n+1} - z{n+1}) + // + + // x update: + // + // x{n+1} = argmin 1/2 ( + // kx |I x - x_t|^2 + + // kv |V x - v_t|^2 + + // ka |A x - a_t|^2 + + // ρ |S x - d|^2 + // ) + // d = (z{n} - u{n}) + + // Rewrite in a friendlier way: + // |A x - b|^2 = x^T A^T A x - 2 x^T A^T b + C + // 1/2 ( + // kx (x^T x - 2 x^T x_t) + + // kv (x^T V^T V x - 2 x^T V^T v_t) + + // ka (x^T A^T A x - 2 x^T A^T a_t) + + // ρ (x^T S^T S x - 2 x^T S^T d) + // ) + C + // + // 1/2 x^T (kx I + kv V^T V + ka A^T A + ρ S^T S) x + // - x^T (kx x_t + kv V^T v_t + ka A^T a_t + ρ S^T d) + // + C + // + // voila: + // M = kx I + kv V^T V + ka A^T A + ρ S^T S + // r = kx x_t + kv V^T v_t + ka A^T a_t + ρ S^T d + // E = 1/2 x^T M x - x^T r + C + + /* + r = ( + torch.matmul(self.N, x_t - x_o_filtered) + + self.admm_stepsize * torch.matmul(self.S.T, - u + z) + ) + */ + Eigen::MatrixXd x_diffs(x_t); + for(auto i : m_constrained_locs) + { + for(uint32_t j = 0; j < numCols; ++j) + { + x_diffs(i, j) = x_diffs(i,j) - x_o(i,j); + } + } + + Eigen::MatrixXd r = m_N * x_diffs; + + for(uint32_t i = 0; i < m_margin_locs.size(); ++i) + { + uint32_t ifull = m_margin_locs[i]; + for(uint32_t j = 0; j < numCols; ++j) + { + r(ifull, j) = r(ifull, j) + m_admm_stepsize * (z(i,j) - u(i,j)); + } + } + + // Solve with respect to pin constraints: + // x = U x_r + x_o + // E = 1/2 (U x_r + x_o)^T M (U x_r + x_o) - (U x_r + x_o)^T r + C + // E = 1/2 (x_r^T U^T + x_o^T) M (U x_r + x_o) - (x_r^T U^T + x_o^T) r + C + // E = 1/2 (x_r^T U^T M (U x_r + x_o) + x_o^T M (U x_r + x_o)) - x_r^T U^T r - x_o^T r + C + // E = 1/2 (x_r^T U^T M U x_r) + x_r^T U^T (M x_o - r) + C + + // minimized when x_r solves this equation: + // U^T M U x_r + U^T (M x_o - r) = 0 + // x_r = (U^T M U)^-1 U^T (r - M x_o) + + // collapse r down to unconstrained variable set: + // rhs = torch.matmul(self.U.T, r) + + uint32_t numRows_reduced = m_unconstrained_locs.size(); + Eigen::MatrixXd r_reduced(numRows_reduced, numCols); + for(uint32_t i = 0; i < numRows_reduced; ++i) + { + uint32_t ifull = m_unconstrained_locs[i]; + for(uint32_t j = 0; j < numCols; ++j) + { + r_reduced(i,j) = r(ifull, j); + } + } + + // solve system: + // x_r = torch.matmul(self.lhsmat_inv, rhs) + r_reduced.conservativeResize(r_reduced.rows(), r_reduced.cols()); + + Eigen::MatrixXd result; + if(m_unconstrained_locs.size()) + { + result = m_system_lu.solve(r_reduced); + } + + // map back to full variable set: + // return torch.matmul(self.U, x_r) + x_o_filtered + for(uint32_t i = 0; i < numRows_reduced; ++i) + { + uint32_t ifull = m_unconstrained_locs[i]; + for(uint32_t j = 0; j < numCols; ++j) + { + x(ifull, j) = result(i, j); + } + } + for(auto i : m_constrained_locs) + { + for(uint32_t j = 0; j < numCols; ++j) + { + x(i, j) = x_o(i, j); + } + } +} + +void TrajectoryCorrector::z_update( + Eigen::MatrixXd &z, + const Eigen::MatrixXd &x, + const Eigen::MatrixXd &z_t, + const Eigen::MatrixXd &u +) const +{ + uint32_t numCols = uint32_t(z.cols()); + + for(uint32_t i = 0; i < m_margin_locs.size(); ++i) + { + + // z_diffs = S x + u - z_t + uint32_t ifull = m_margin_locs[i]; + for(uint32_t j = 0; j < numCols; ++j) + { + z(i, j) = x(ifull, j) + u(i, j) - z_t(i, j); + } + + // find the norm of the current z diff vector: + double z_diff_norm = 0.0; + for(uint32_t j = 0; j < numCols; ++j) + { + double z_diff = z(i, j); + z_diff_norm += z_diff * z_diff; + } + z_diff_norm = sqrt(z_diff_norm); + + // if the norm is greater than the margin size, we need to rescale + // the diff: + if( z_diff_norm > m_margin_vals[i] ) + { + for(uint32_t j = 0; j < numCols; ++j) + { + z(i, j) = z(i, j) * m_margin_vals[i] / z_diff_norm; + } + } + + // add the diff back on to the target: + for(uint32_t j = 0; j < numCols; ++j) + { + z(i, j) = z_t(i, j) + z(i, j); + } + } +} + +void TrajectoryCorrector::u_update( + Eigen::MatrixXd &u, + const Eigen::MatrixXd &x, + const Eigen::MatrixXd &z +) const +{ + uint32_t numCols = uint32_t(z.cols()); + + // u += S x - z + for(uint32_t i = 0; i < m_margin_locs.size(); ++i) + { + uint32_t ifull = m_margin_locs[i]; + for(uint32_t j = 0; j < numCols; ++j) + { + u(i,j) += x(ifull, j) - z(i,j); + } + } +} diff --git a/MotionCorrection/src/cpp/AnimProcessing/TrajectoryCorrector.h b/MotionCorrection/src/cpp/AnimProcessing/TrajectoryCorrector.h new file mode 100644 index 0000000000000000000000000000000000000000..9e0a4fb74a386caace0e0bcd542795180c8b795b --- /dev/null +++ b/MotionCorrection/src/cpp/AnimProcessing/TrajectoryCorrector.h @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +class TrajectoryCorrector +{ +public: + + static void computeDiffMats( + Eigen::SparseMatrix& V, + Eigen::SparseMatrix& A, + uint32_t N, + const Eigen::VectorXd& velocityWeights = Eigen::VectorXd(), + Eigen::MatrixXd* v_rhs = nullptr, + Eigen::MatrixXd* a_rhs = nullptr + ); + + TrajectoryCorrector( + const Eigen::VectorXd& margins, + float pos_weight, + float vel_weight, + float acc_weight, + const Eigen::VectorXd& velocityWeights = Eigen::VectorXd(), + uint32_t admm_iters=100 ); + + void Interpolate( + Eigen::MatrixXd& ret, + const Eigen::MatrixXd& observations, + const Eigen::MatrixXd& ref_positions + ) const; + + void x_update( + Eigen::MatrixXd& x, + const Eigen::MatrixXd& z, + const Eigen::MatrixXd& u, + const Eigen::MatrixXd& x_t, + const Eigen::MatrixXd& x_o + ) const; + + void z_update( + Eigen::MatrixXd& z, + const Eigen::MatrixXd& x, + const Eigen::MatrixXd& z_t, + const Eigen::MatrixXd& u + ) const; + + void u_update( + Eigen::MatrixXd& u, + const Eigen::MatrixXd& x, + const Eigen::MatrixXd& z + ) const; + + float admm_stepsize() const { return m_admm_stepsize; } + + const std::vector& margin_locs() { return m_margin_locs; } + +private: + + Eigen::SparseMatrix m_N; + Eigen::SparseLU> m_system_lu; + + uint32_t m_admm_iters; + + std::vector m_margin_locs; + std::vector m_margin_vals; + + std::vector m_unconstrained_locs; + std::vector m_constrained_locs; + + float m_admm_stepsize; + +}; diff --git a/MotionCorrection/src/cpp/AnimProcessing/Utility.cpp b/MotionCorrection/src/cpp/AnimProcessing/Utility.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ce7f6453770c5a832942ef4e345cabfdfa0641e7 --- /dev/null +++ b/MotionCorrection/src/cpp/AnimProcessing/Utility.cpp @@ -0,0 +1,1197 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "TrajectoryCorrector.h" +#include "InverseKinematics.h" + +#include "Utility.h" + +#include +#include +#include +#include +#include +using Pose = std::vector; + +static const float pos_weight = 0.001f; +static const float vel_weight = 1.0f; +static const float acc_weight = 10.0f; + + +namespace { + + // Enable with: MOTIONCORRECTION_DEBUG_INTERVALS=1 + // Default: off (no Interval printing). + bool DebugPrintIntervalsEnabled() + { + const char* v = std::getenv("MOTIONCORRECTION_DEBUG_INTERVALS"); + if (v == nullptr || v[0] == '\0') + { + return false; + } + // Treat "0" as false; any other non-empty value enables. + return v[0] != '0'; + } + + + void FilterContactIntervals( + std::vector>& contactIntervals, + const std::vector& mask, + bool one_bone_contact = false) + { + std::vector keepIntervals; + for (size_t i = 0; i < contactIntervals.size(); ++i) + { + const auto& interval = contactIntervals[i]; + + bool startConstrained = (interval.first != 0 && mask[interval.first - 1]); + bool endConstrained; + + endConstrained = (interval.second != mask.size() && mask[interval.second]); + + if (one_bone_contact) + { + if (startConstrained || endConstrained) + { + continue; + } + } + else + { + // If both the start and end of the contact interval are masked, + // there's no way we can correct the contact without popping, so + // let's filter these out: + if (startConstrained && endConstrained) + { + continue; + } + } + + keepIntervals.push_back(i); + } + + for (size_t i = 0; i < keepIntervals.size(); ++i) + { + contactIntervals[i] = contactIntervals[keepIntervals[i]]; + } + contactIntervals.resize(keepIntervals.size()); + } + + std::vector> ComputeContactIntervals( + const std::vector& contacts, + const std::vector& mask, + float contactThreshold) + { + // turn off the contacts for all frames that are constrained/masked: + std::vector contactsNoMask = contacts; + for (size_t i = 0; i < mask.size(); ++i) + { + if (mask[i]) + { + contactsNoMask[i] = 0; + } + } + + // Find intervals that are in contact: + std::vector> contactIntervals; + int start = -1; + for (int frame = 0; frame < mask.size(); ++frame) + { + bool isContact = contactsNoMask[frame] > contactThreshold; + if (isContact && start == -1) + { + start = frame; + } + else if (!isContact && start != -1) + { + contactIntervals.emplace_back(start, frame); + start = -1; + } + } + + // Close the final interval if needed: + if (start != -1) + { + contactIntervals.emplace_back(start, mask.size()); + } + return contactIntervals; + } + + void FindContactPoints( + std::vector &points, + std::vector &inContact, + const std::vector& joint_parents_vec, + int32_t jointIndex, + const std::vector &poses, + const std::vector>& contactIntervals, + const std::vector& mask, + size_t frameCount, + float minHeight) + { + // Find a representative frame for each interval. + // If the interval starts after a masked frame, use the start + // of the interval, if it ends before a mask use the end, + // otherwise use the middle frame. + inContact.clear(); + inContact.resize(frameCount, 0); + points.clear(); + points.resize(frameCount); + for (size_t i = 0; i < contactIntervals.size(); ++i) + { + const auto& interval = contactIntervals[i]; + int frame = -1; + bool startConstrained = (interval.first != 0 && mask[interval.first - 1]); + bool endConstrained; + + endConstrained = (interval.second != mask.size() && mask[interval.second]); + + // Debug output (opt-in via env var) + if (DebugPrintIntervalsEnabled()) + { + std::cout << "Interval " << i << ": start=" << interval.first + << ", end=" << interval.second + << ", startConstrained=" << startConstrained + << ", endConstrained=" << endConstrained << std::endl; + } + + if(startConstrained) + { + // If the interval starts on a constraint, use the constrained frame + // as a target (doing this modulo mask.size() in case we're looping) + frame = interval.first - 1; + } + else if (endConstrained) + { + // If the interval ends on a constraint, use the constrained frame + // as a target: + frame = interval.second; + } + else + { + // Otherwise use the midpoint of the interval: + frame = (interval.first + interval.second) / 2; + } + + // get the target point: + Math::Vector target = Animation::JointLocalToGlobal(joint_parents_vec, jointIndex, poses[frame]).GetTranslation(); + for(int i = interval.first; i < interval.second; ++i) + { + Math::Vector framePt = Animation::JointLocalToGlobal(joint_parents_vec, jointIndex, poses[i]).GetTranslation(); + inContact[i] = 1; + points[i] = target; + if (!startConstrained && !endConstrained) + { + points[i].SetY(std::max(framePt.GetY(), minHeight)); + // std::cout << " Frame " << i << ": SetY with framePt.GetY()=" << framePt.GetY() + // << ", minHeight=" << minHeight << std::endl; + } + } + } + } + + float TargetReachFalloff( + const std::vector& joint_parents_vec, + const Pose& defaultPose, + int32_t jointIndex, + Animation::IKType ikType, + const Math::Vector& target, + const Pose& pose, + const Math::Transform& rootTx = Math::Transform::Identity) + { + float maxReach = defaultPose[jointIndex].GetTranslation().GetLength3(); + if (ikType == Animation::IKType::kTwoBone) + { + jointIndex = joint_parents_vec[jointIndex]; + ASSERT(jointIndex > -1); + maxReach += defaultPose[jointIndex].GetTranslation().GetLength3(); + } + // Get base joint world Tx + jointIndex = joint_parents_vec[jointIndex]; + ASSERT(jointIndex > -1); + const auto worldTx = Animation::JointLocalToGlobal(joint_parents_vec, jointIndex, pose, rootTx); + + // Gaussian falloff + float targetDist = target.GetDistance3(worldTx.GetTranslation()); + float tmp = Math::Max(targetDist / maxReach - 0.99f, 0.f) / 0.01f; + tmp = tmp * tmp; + return std::exp(-2.f * tmp * tmp); + } + + void CorrectHipsY( + std::vector& poses, + const std::vector& targetPoses, + const std::vector& fullBodyMask, + const std::vector& contacts, + float contactThreshold + ) + { + // Correct the y coordinates of the root. + auto N = poses.size(); + Eigen::MatrixXd x(N, 1); + Eigen::MatrixXd observations(N, 1); + Eigen::MatrixXd xfixed(N, 1); + + // Fill in the initial trajectory (x) and the values we want to hit when we + // warp it (observations): + Eigen::VectorXd yCorrectMargins(N); + for(size_t frame = 0; frame < N; ++frame) + { + yCorrectMargins[frame] = fullBodyMask[frame] ? 0.0f : -1.0f; + x(frame, 0) = ((float*)&poses[frame][0].GetTranslation())[1]; + observations(frame, 0) = ((float*)&targetPoses[frame][0].GetTranslation())[1]; + } + + TrajectoryCorrector ycorrector( + yCorrectMargins, + pos_weight * 10, + vel_weight, + acc_weight * 0.1f + ); + ycorrector.Interpolate( + xfixed, + observations, + x + ); + + // fill channel again: + for (uint32_t frame = 0; frame < N; ++frame) + { + ((float*)&poses[frame][0].GetTranslation())[1] = float(xfixed(frame, 0)); + } + } + + void SmoothChannels( + Eigen::MatrixXd &x, + const std::vector& mask + ) + { + for( uint32_t i=0; i < mask.size(); ++i) + { + uint32_t i_prev = i == 0 ? 0 : i-1; + uint32_t i_next = std::min(uint32_t(i+1), uint32_t(mask.size()-1)); + if(i > 0 && mask[i] > 0 && mask[i_prev] == 0) + { + // if the previous frame is unconstrained and the current frame is constrained, + // replace the current frame with the average of its neighbors: + for(long j=0; j < x.cols(); ++j) + { + x(i, j) = 0.5f * (x(i_prev, j) + x(i_next, j)); + } + } + if(mask[i] > 0 && mask[i_next] == 0) + { + // if the next frame is unconstrained and the current frame is constrained, + // replace the current frame with the average of its neighbors: + for(long j=0; j < x.cols(); ++j) + { + x(i, j) = 0.5f * (x(i_prev, j) + x(i_next, j)); + } + } + } + } + + + void CorrectHipsXZ( + std::vector& poses, + const std::vector& targetPoses, + const std::vector& fullBodyMask, + const std::vector& rootMask, + const std::vector& endEffectorPins, + const Eigen::VectorXd& velocity_weights, + float root_margin + ) + { + auto N = poses.size(); + Eigen::VectorXd margins(N); + for( size_t i = 0; i < N; ++i ) + { + margins[i] = fullBodyMask[i] ? 0.0f : -1.0f; + } + + std::vector rootCombinedMask(N, 0.0f); + for(size_t i = 0; i < N; ++i) + { + rootCombinedMask[i] = (fullBodyMask[i] > 0) || (rootMask[i] > 0); + if(rootMask[i] > 0 && margins[i] != 0) + { + margins[i] = root_margin; + } + for (auto& c : endEffectorPins) + { + if (c.contactMask[i] && margins[i] != 0) + { + margins[i] = root_margin; + } + } + } + TrajectoryCorrector xzcorrector( + margins, + pos_weight, + vel_weight, + acc_weight, + velocity_weights + ); + + // Enforce pose constraints on root xz trajectory: + Eigen::MatrixXd x(N, 2); + Eigen::MatrixXd observations(N, 2); + Eigen::MatrixXd x_fixed(N, 2); + + observations.setZero(); + for (uint32_t frame = 0; frame < N; ++frame) + { + x(frame, 0) = ((float*)&poses[frame][0].GetTranslation())[0]; + x(frame, 1) = ((float*)&poses[frame][0].GetTranslation())[2]; + + observations(frame, 0) = ((float*)&targetPoses[frame][0].GetTranslation())[0]; + observations(frame, 1) = ((float*)&targetPoses[frame][0].GetTranslation())[2]; + } + + SmoothChannels(x, rootCombinedMask); + + xzcorrector.Interpolate( + x_fixed, + observations, + x + ); + + // fill channels again: + for (uint32_t frame = 0; frame < N; ++frame) + { + ((float*)&poses[frame][0].GetTranslation())[0] = float(x_fixed(frame, 0)); + ((float*)&poses[frame][0].GetTranslation())[2] = float(x_fixed(frame, 1)); + } + } + + void CorrectRotationsForBone( + std::vector& poses, + const std::vector& targetPoses, + const std::vector& mask, + const TrajectoryCorrector& corrector, + int boneIdx, + bool performChannelSmoothing) + { + auto N = poses.size(); + Eigen::MatrixXd x(N, 1); + Eigen::MatrixXd observations(N, 1); + observations.setZero(); + Eigen::MatrixXd x_fixed(N, 1); + + // Quaternion components can flip when they pass through 180 degree + // rotations, so let's convert all the quaternions in this channel to + // the forward/up vector representation, modify them, then convert back + // to quaternions: + + // convert time series to 6d forward/up: + std::vector forwardUp(6 * N); + std::vector targetForwardUp(6 * N); + for (uint32_t frame = 0; frame < N; ++frame) + { + auto q = poses[frame][boneIdx].GetRotation(); + auto forward = q.ZAxis(); + auto up = q.YAxis(); + forwardUp[N * 0 + frame] = forward.GetX(); + forwardUp[N * 1 + frame] = forward.GetY(); + forwardUp[N * 2 + frame] = forward.GetZ(); + forwardUp[N * 3 + frame] = up.GetX(); + forwardUp[N * 4 + frame] = up.GetY(); + forwardUp[N * 5 + frame] = up.GetZ(); + + q = targetPoses[frame][boneIdx].GetRotation(); + forward = q.ZAxis(); + up = q.YAxis(); + targetForwardUp[N * 0 + frame] = forward.GetX(); + targetForwardUp[N * 1 + frame] = forward.GetY(); + targetForwardUp[N * 2 + frame] = forward.GetZ(); + targetForwardUp[N * 3 + frame] = up.GetX(); + targetForwardUp[N * 4 + frame] = up.GetY(); + targetForwardUp[N * 5 + frame] = up.GetZ(); + } + + // correct trajectories: + for (uint32_t dim = 0; dim < 6; ++dim) + { + for (uint32_t frame = 0; frame < N; ++frame) + { + x(frame, 0) = forwardUp[N * dim + frame]; + observations(frame, 0) = mask[frame] * targetForwardUp[N * dim + frame]; + } + + if (performChannelSmoothing) + { + SmoothChannels(x, mask); + } + + corrector.Interpolate( + x_fixed, + observations, + x + ); + + // fill channel again: + for (uint32_t frame = 0; frame < N; ++frame) + { + forwardUp[N * dim + frame] = float(x_fixed(frame, 0)); + } + } + + for (uint32_t frame = 0; frame < N; ++frame) + { + Math::Vector forward = { forwardUp[N * 0 + frame] ,forwardUp[N * 1 + frame] ,forwardUp[N * 2 + frame] }; + Math::Vector up = { forwardUp[N * 3 + frame] ,forwardUp[N * 4 + frame] ,forwardUp[N * 5 + frame] }; + + forward.Normalize3(); + up.Normalize3(); + + poses[frame][boneIdx].SetRotation(Math::Quaternion::LookRotation(forward, up)); + } + } + + void CorrectJointRotations( + std::vector& poses, + const std::vector& targetPoses, + const std::vector& fullBodyMask, + const Eigen::VectorXd& velocity_weights + ) + { + auto N = poses.size(); + + // Create a trajectory corrector for fixing the full body fullBodyMask positions: + Eigen::VectorXd margins(N); + for( size_t i = 0; i < N; ++i ) + { + margins[i] = fullBodyMask[i] ? 0.0f : -1.0f; + } + TrajectoryCorrector corrector( + margins, + pos_weight * 10, + vel_weight, + acc_weight, + velocity_weights + ); + + for (uint32_t boneIdx = 0; boneIdx < poses[0].size(); ++boneIdx) + { + CorrectRotationsForBone( + poses, + targetPoses, + fullBodyMask, + corrector, + boneIdx, + true + ); + } + } + + void DoEffectorIK( + std::vector& poses, + const std::vector& targetPoses, + const std::vector& fullBodyMask, + const std::vector& endEffectorPins, + const std::vector& joint_parents_vec, + const std::vector& defaultPose + ) + { + // Apply IK for effector pins + auto N = poses.size(); + std::map> jointCorrectionMasks; + std::vector ikFixedPoses = poses; + for (auto& c : endEffectorPins) + { + auto jointIdx = c.jointIndex; + + if(jointCorrectionMasks[jointIdx].empty()) + { + // initialize to the full body constraint mask because we + // want to constrain that anyway: + jointCorrectionMasks[jointIdx] = fullBodyMask; + } + + // Add a trajectory correction mask for the parent joint: + auto parentIdx = joint_parents_vec[jointIdx]; + if(jointCorrectionMasks[parentIdx].empty()) + { + // initialize to the full body constraint mask because we + // want to constrain that anyway: + jointCorrectionMasks[parentIdx] = fullBodyMask; + } + + // Add a trajectory correction mask for its parent if this is + // 2 bone IK: + auto parentParentIdx = joint_parents_vec[parentIdx]; + if(c.contactType == Animation::kTwoBone) + { + if(jointCorrectionMasks[parentParentIdx].empty()) + { + // initialize to the full body constraint mask because we + // want to constrain that anyway: + jointCorrectionMasks[parentParentIdx] = fullBodyMask; + } + } + + for (uint32_t fixFrame = 0; fixFrame < fullBodyMask.size(); ++fixFrame) + { + if (c.contactMask[fixFrame]) + { + const auto targetGlobalTransform = Animation::JointLocalToGlobal(joint_parents_vec, jointIdx, targetPoses[fixFrame]); + + // flag the parent joint as fixed in its correction mask: + jointCorrectionMasks[parentIdx][fixFrame] = 1; + switch(c.contactType) + { + case Animation::kOneBone: + { + IK::OneBoneIk( + ikFixedPoses[fixFrame], + Math::Transform::Identity, + jointIdx, + 1.0, + targetGlobalTransform.GetTranslation(), + joint_parents_vec + ); + break; + } + case Animation::kTwoBone: + { + // flag the parent parent joint as fixed in its correction mask: + jointCorrectionMasks[parentParentIdx][fixFrame] = 1; + IK::TwoBoneIk( + ikFixedPoses[fixFrame], + Math::Transform::Identity, + jointIdx, + 1.0, + targetGlobalTransform.GetTranslation(), + joint_parents_vec, + c.hintOffset + ); + break; + } + } + + // now we need to fix things so the global rotation of the joint + // matches the input: + jointCorrectionMasks[jointIdx][fixFrame] = 1; + auto parentGlobalTransform = Animation::JointLocalToGlobal(joint_parents_vec, parentIdx, ikFixedPoses[fixFrame]); + ikFixedPoses[fixFrame][jointIdx].SetRotation( + targetGlobalTransform.GetRotation() * parentGlobalTransform.GetRotation().GetConjugate() + ); + + } + } + } + + // Applying the effector pin IK introduces popping into the animation, + // so let's apply the interpolator to all the joints we modified so as to + // line the trajectory up properly again: + Eigen::VectorXd margins(N); + for( auto &kv : jointCorrectionMasks) + { + for( size_t i = 0; i < N; ++i ) + { + margins[i] = kv.second[i] ? 0.0f : -1.0f; + } + TrajectoryCorrector corrector(margins, pos_weight * 10, vel_weight, acc_weight); + + CorrectRotationsForBone( + poses, + ikFixedPoses, + kv.second, + corrector, + kv.first, + false + ); + } + } + + void DoContactIK( + std::vector& poses, + const std::vector& fullBodyMask, + const std::vector& contacts, + const std::vector& endEffectorPins, + const std::vector& joint_parents_vec, + const std::vector& defaultPose, + float contactThreshold, + bool has_double_ankle_joints + ) + { + auto N = poses.size(); + Eigen::VectorXd margins = Eigen::VectorXd::Zero(N); + + // Apply IK to stabilize limbs on contacts + std::map> jointCorrectionMasks; + std::vector ikFixedPoses = poses; + + // Save original poses before any modifications (for double ankle correction later) + const std::vector originalPoses = poses; + + // Track which frames were corrected for each 2-bone contact (for double ankle correction later) + std::map> twoBoneContactFrames; + + auto addEndEffectorMask = [&](uint32_t jointIdx, uint32_t parentIdx, std::vector& jointMask) + { + auto it = std::find_if( + endEffectorPins.begin(), endEffectorPins.end(), + [&](const auto &c) + { + if(jointIdx == c.jointIndex) + { + return true; + } + return false; + } + ); + if(it == endEffectorPins.end()) + { + // We could be correcting the toe joint, in which case we need to use + // the parent joint instead: + it = std::find_if( + endEffectorPins.begin(), endEffectorPins.end(), + [&](const auto &c) + { + if(parentIdx == c.jointIndex) + { + return true; + } + return false; + } + ); + } + if(it != endEffectorPins.end()) + { + const auto &msk = it->contactMask; + for(size_t i=0; i < msk.size(); ++i) + { + if(msk[i]) + { + jointMask[i] = 1.0f; + } + } + } + }; + + // Process two bone contacts first: + for (auto& c : contacts) + { + if(c.contactType != Animation::kTwoBone) + { + continue; + } + const auto jointIdx = c.jointIndex; + auto parentIdx = joint_parents_vec[jointIdx]; + auto parentParentIdx = joint_parents_vec[parentIdx]; + + auto jointMask = fullBodyMask; + addEndEffectorMask(jointIdx, parentIdx, jointMask); + + // We'll actually be modifying 3 joints here: + // * The two joints immediately up in the hierarchy because of the 2 bone IK + // * The joint itself because we restore its original global rotation + if(jointCorrectionMasks[parentIdx].empty()) + { + jointCorrectionMasks[parentIdx] = jointMask; + } + if(jointCorrectionMasks[parentParentIdx].empty()) + { + jointCorrectionMasks[parentParentIdx] = jointMask; + } + if(jointCorrectionMasks[jointIdx].empty()) + { + jointCorrectionMasks[jointIdx] = jointMask; + } + + // Compute the intervals in which the joint is in contact with the floor: + auto contactIntervals = ComputeContactIntervals(c.contactMask, jointMask, contactThreshold); + FilterContactIntervals(contactIntervals, jointMask); + + std::vector contactPoints; + std::vector inContact; + FindContactPoints( + contactPoints, + inContact, + joint_parents_vec, + jointIdx, + poses, + contactIntervals, + jointMask, + c.contactMask.size(), + c.minHeight + ); + + for (uint32_t fixFrame = 0; fixFrame < fullBodyMask.size(); ++fixFrame) + { + if (inContact[fixFrame]) + { + auto target = contactPoints[fixFrame]; + jointCorrectionMasks[parentIdx][fixFrame] = 1.0f; + jointCorrectionMasks[parentParentIdx][fixFrame] = 1.0f; + jointCorrectionMasks[jointIdx][fixFrame] = 1.0f; + + // Track this frame for double ankle correction later + if (has_double_ankle_joints) + { + if (twoBoneContactFrames[jointIdx].empty()) + twoBoneContactFrames[jointIdx].resize(fullBodyMask.size(), false); + twoBoneContactFrames[jointIdx][fixFrame] = true; + } + + // save the original global rotation of the joint: + auto jointGlobalRotation = Animation::JointLocalToGlobal( + joint_parents_vec, + jointIdx, + ikFixedPoses[fixFrame] + ).GetRotation(); + + const float w = TargetReachFalloff( + joint_parents_vec, + defaultPose, + jointIdx, + c.contactType, + target, + ikFixedPoses[fixFrame] + ); + // std::cout << "Frame " << fixFrame << ": w=" << w << std::endl; + + // apply the 2 bone IK: + auto origParentRotation = ikFixedPoses[fixFrame][parentIdx].GetRotation(); + auto origParentParentRotation = ikFixedPoses[fixFrame][parentParentIdx].GetRotation(); + IK::TwoBoneIk( + ikFixedPoses[fixFrame], + Math::Transform::Identity, + jointIdx, + 1.0f, + target, + joint_parents_vec, + c.hintOffset + ); + ikFixedPoses[fixFrame][parentIdx].SetRotation(Math::Quaternion::SLerp(origParentRotation, ikFixedPoses[fixFrame][parentIdx].GetRotation(), w)); + ikFixedPoses[fixFrame][parentParentIdx].SetRotation(Math::Quaternion::SLerp(origParentParentRotation, ikFixedPoses[fixFrame][parentParentIdx].GetRotation(), w)); + + // restore previous global rotation of this joint: + auto parentGloblalRotation = Animation::JointLocalToGlobal( + joint_parents_vec, + parentIdx, + ikFixedPoses[fixFrame] + ).GetRotation(); + + jointCorrectionMasks[jointIdx][fixFrame] = 1.0f; + ikFixedPoses[fixFrame][jointIdx].SetRotation( + jointGlobalRotation * parentGloblalRotation.GetConjugate() + ); + + auto result = Animation::JointLocalToGlobal( + joint_parents_vec, + jointIdx, + ikFixedPoses[fixFrame] + ).GetTranslation(); + } + } + + } + + for( auto &kv : jointCorrectionMasks) + { + for( size_t i = 0; i < N; ++i ) + { + margins[i] = kv.second[i] ? 0.0f : -1.0f; + } + TrajectoryCorrector corrector(margins, pos_weight * 10, vel_weight, acc_weight); + CorrectRotationsForBone( + poses, + ikFixedPoses, + kv.second, + corrector, + kv.first, + false + ); + } + jointCorrectionMasks.clear(); + + // Then process one bone contacts: + for(auto &c : contacts) + { + if(c.contactType != Animation::kOneBone) + { + continue; + } + const auto jointIdx = c.jointIndex; + auto parentIdx = joint_parents_vec[jointIdx]; + + // We can't touch frames that have been constrained with full body constraints + // or the end effector constraints for this joint, so let's combine fullBodyMask + // with the end effector mask for this joint if it exists so we can use that + // information later: + auto jointMask = fullBodyMask; + addEndEffectorMask(jointIdx, parentIdx, jointMask); + + // Add a trajectory correction mask for the parent joint: + if(jointCorrectionMasks[parentIdx].empty()) + { + jointCorrectionMasks[parentIdx] = jointMask; + } + + // Compute the intervals in which the joint is in contact with the floor: + auto contactIntervals = ComputeContactIntervals(c.contactMask, jointMask, contactThreshold); + FilterContactIntervals(contactIntervals, jointMask, true); + for(const auto &interval : contactIntervals) + { + for (int fixFrame = interval.first; fixFrame < interval.second; ++fixFrame) + { + // All we're going to do here is stick the joint to the floor - + // we're going to allow it to slide from side to side. + + // Find a target position that lies on the floor by iteratively + // projecting the joint to the floor (pure laziness really, this could + // be done analytically): + Math::Vector parentPos = Animation::JointLocalToGlobal(joint_parents_vec, parentIdx, ikFixedPoses[fixFrame]).GetTranslation(); + Math::Vector target = Animation::JointLocalToGlobal(joint_parents_vec, jointIdx, ikFixedPoses[fixFrame]).GetTranslation(); + float jointLength = (target - parentPos).GetLength3(); + for(int32_t i = 0; i < 10; ++i) + { + target.SetY(c.minHeight); + auto dir = (target - parentPos).GetNormalized3(); + target = parentPos + dir * jointLength; + } + + IK::OneBoneIk( + ikFixedPoses[fixFrame], + Math::Transform::Identity, + jointIdx, + 1.0f, + target, + joint_parents_vec + ); + jointCorrectionMasks[parentIdx][fixFrame] = 1.0f; + } + } + + } + + // Fixing the contacts with IK will introduce popping into the animation, + // so let's apply the interpolator to all the joints we modified so as to + // line the trajectory up properly again: + for( auto &kv : jointCorrectionMasks) + { + for( size_t i = 0; i < N; ++i ) + { + margins[i] = kv.second[i] ? 0.0f : -1.0f; + } + TrajectoryCorrector corrector(margins, pos_weight * 10, vel_weight, acc_weight); + CorrectRotationsForBone( + poses, + ikFixedPoses, + kv.second, + corrector, + kv.first, + false + ); + } + + if (has_double_ankle_joints) + { + // Maps to save target positions BEFORE 2-bone IK modifies them + std::map> savedFirstAnkleTargets; // [firstAnkleIdx][frame] -> position + std::map> savedToeTargets; // [firstAnkleIdx][frame] -> position + std::map contactToToeIdx; // firstAnkleIdx -> toeIdx + + // Find toe joints for each leg + for (const auto& tc : contacts) + { + if (tc.contactType == Animation::kOneBone) + { + // The parent of the toe is the 1st ankle + int parentIdx = joint_parents_vec[tc.jointIndex]; + if (parentIdx >= 0) + { + contactToToeIdx[parentIdx] = tc.jointIndex; + } + } + } + + // For each 2-bone contact, correct the parent (2nd ankle) joint + for (auto& c : contacts) + { + if (c.contactType != Animation::kTwoBone) + continue; + + const auto firstAnkleIdx = c.jointIndex; + const auto secondAnkleIdx = joint_parents_vec[firstAnkleIdx]; + const auto kneeIdx = joint_parents_vec[secondAnkleIdx]; + const auto hipIdx = joint_parents_vec[kneeIdx]; + + if (hipIdx < 0) continue; // safety check + + // Get saved contact frames for this ankle + auto it = twoBoneContactFrames.find(firstAnkleIdx); + if (it == twoBoneContactFrames.end()) + continue; + const auto& contactFrames = it->second; + + // Add correction mask for knee and hip + auto jointMask = fullBodyMask; + addEndEffectorMask(firstAnkleIdx, secondAnkleIdx, jointMask); + + if (jointCorrectionMasks[kneeIdx].empty()) + jointCorrectionMasks[kneeIdx] = jointMask; + if (jointCorrectionMasks[hipIdx].empty()) + jointCorrectionMasks[hipIdx] = jointMask; + + for (uint32_t fixFrame = 0; fixFrame < fullBodyMask.size(); ++fixFrame) + { + // Only correct frames where the 1st ankle was corrected + if (!contactFrames[fixFrame]) + continue; + + // *** SAVE TARGET POSITIONS BEFORE 2-BONE IK *** + savedFirstAnkleTargets[firstAnkleIdx][fixFrame] = Animation::JointLocalToGlobal( + joint_parents_vec, firstAnkleIdx, ikFixedPoses[fixFrame]).GetTranslation(); + + if (contactToToeIdx.count(firstAnkleIdx)) + { + savedToeTargets[firstAnkleIdx][fixFrame] = Animation::JointLocalToGlobal( + joint_parents_vec, contactToToeIdx[firstAnkleIdx], ikFixedPoses[fixFrame]).GetTranslation(); + } + + // Get original global transforms (before any IK corrections) + auto originalFirstAnkleGlobal = Animation::JointLocalToGlobal( + joint_parents_vec, firstAnkleIdx, originalPoses[fixFrame]); + auto originalSecondAnkleGlobal = Animation::JointLocalToGlobal( + joint_parents_vec, secondAnkleIdx, originalPoses[fixFrame]); + + // Compute delta from 1st ankle to 2nd ankle in original animation + auto deltaFirstToSecond = originalFirstAnkleGlobal.GetDeltaToOther(originalSecondAnkleGlobal); + + // Get corrected 1st ankle global transform + auto correctedFirstAnkleGlobal = Animation::JointLocalToGlobal( + joint_parents_vec, firstAnkleIdx, ikFixedPoses[fixFrame]); + + // Apply the original delta to the corrected 1st ankle to get target for 2nd ankle + auto target = (deltaFirstToSecond * correctedFirstAnkleGlobal).GetTranslation(); + + // print current and target second ankle positions + auto currPos = Animation::JointLocalToGlobal( + joint_parents_vec, secondAnkleIdx, ikFixedPoses[fixFrame]).GetTranslation(); + + // Apply 2-bone IK: Hip -> Knee -> 2nd Ankle + IK::TwoBoneIk( + ikFixedPoses[fixFrame], + Math::Transform::Identity, + secondAnkleIdx, + 1.0f, + target, + joint_parents_vec, + c.hintOffset + ); + + // auto correctedPos = Animation::JointLocalToGlobal( + // joint_parents_vec, secondAnkleIdx, ikFixedPoses[fixFrame]).GetTranslation(); + // std::cout << "Frame " << fixFrame << ": target second ankle=(" << target.GetX() << ", " << target.GetY() << ", " << target.GetZ() << "), corrected second ankle position=(" << correctedPos.GetX() << ", " << correctedPos.GetY() << ", " << correctedPos.GetZ() << ")" << std::endl; + + jointCorrectionMasks[kneeIdx][fixFrame] = 1.0f; + jointCorrectionMasks[hipIdx][fixFrame] = 1.0f; + } + } + + // Smooth the corrected joints + for (auto& kv : jointCorrectionMasks) + { + for (size_t i = 0; i < N; ++i) + margins[i] = kv.second[i] ? 0.0f : -1.0f; + + TrajectoryCorrector corrector(margins, pos_weight * 10, vel_weight, acc_weight); + CorrectRotationsForBone(poses, ikFixedPoses, kv.second, corrector, kv.first, false); + } + + // *** PHASE 2: 1-bone IKs to restore 1st ankle and toe *** + jointCorrectionMasks.clear(); + + for (auto& c : contacts) + { + if (c.contactType != Animation::kTwoBone) + continue; + + const auto firstAnkleIdx = c.jointIndex; + const auto secondAnkleIdx = joint_parents_vec[firstAnkleIdx]; + + auto it = twoBoneContactFrames.find(firstAnkleIdx); + if (it == twoBoneContactFrames.end()) + continue; + + // Setup correction masks + auto jointMask = fullBodyMask; + addEndEffectorMask(firstAnkleIdx, secondAnkleIdx, jointMask); + + if (jointCorrectionMasks[secondAnkleIdx].empty()) + jointCorrectionMasks[secondAnkleIdx] = jointMask; + if (jointCorrectionMasks[firstAnkleIdx].empty()) + jointCorrectionMasks[firstAnkleIdx] = jointMask; + + for (uint32_t fixFrame = 0; fixFrame < fullBodyMask.size(); ++fixFrame) + { + if (!it->second[fixFrame]) + continue; + + // 1-bone IK: Rotate 2nd ankle so 1st ankle reaches saved target + IK::OneBoneIk( + ikFixedPoses[fixFrame], + Math::Transform::Identity, + firstAnkleIdx, + 1.0f, + savedFirstAnkleTargets[firstAnkleIdx][fixFrame], + joint_parents_vec + ); + jointCorrectionMasks[secondAnkleIdx][fixFrame] = 1.0f; + + // auto target = savedFirstAnkleTargets[firstAnkleIdx][fixFrame]; + // auto corrected = Animation::JointLocalToGlobal( + // joint_parents_vec, firstAnkleIdx, ikFixedPoses[fixFrame]).GetTranslation(); + // std::cout << "Frame " << fixFrame << ": target first ankle=(" << target.GetX() << ", " << target.GetY() << ", " << target.GetZ() << "), corrected first ankle=(" << corrected.GetX() << ", " << corrected.GetY() << ", " << corrected.GetZ() << ")" << std::endl; + + // 1-bone IK: Rotate 1st ankle so toe reaches saved target + if (contactToToeIdx.count(firstAnkleIdx) && savedToeTargets[firstAnkleIdx].count(fixFrame)) + { + IK::OneBoneIk( + ikFixedPoses[fixFrame], + Math::Transform::Identity, + contactToToeIdx[firstAnkleIdx], + 1.0f, + savedToeTargets[firstAnkleIdx][fixFrame], + joint_parents_vec + ); + jointCorrectionMasks[firstAnkleIdx][fixFrame] = 1.0f; + } + + // target = savedToeTargets[firstAnkleIdx][fixFrame]; + // corrected = Animation::JointLocalToGlobal( + // joint_parents_vec, contactToToeIdx[firstAnkleIdx], ikFixedPoses[fixFrame]).GetTranslation(); + // std::cout << "Frame " << fixFrame << ": target toe=(" << target.GetX() << ", " << target.GetY() << ", " << target.GetZ() << "), corrected toe=(" << corrected.GetX() << ", " << corrected.GetY() << ", " << corrected.GetZ() << ")" << std::endl; + } + } + + // Smooth 2nd ankle and 1st ankle + for (auto& kv : jointCorrectionMasks) + { + for (size_t i = 0; i < N; ++i) + margins[i] = kv.second[i] ? 0.0f : -1.0f; + + TrajectoryCorrector corrector(margins, pos_weight * 10, vel_weight, acc_weight); + CorrectRotationsForBone(poses, ikFixedPoses, kv.second, corrector, kv.first, false); + } + } + } + +} + + +Math::Transform Animation::JointLocalToGlobal( + const std::vector& joint_parents_vec, + int32_t index, + const Pose& localPose, + const Math::Transform& rootTx) +{ + Math::Transform worldTx = Math::Transform::Identity; + while (index > -1) + { + worldTx = worldTx * localPose[index]; + index = joint_parents_vec[index]; + } + + return worldTx * rootTx; +} + +void Animation::CorrectMotion( + std::vector& poses, + const std::vector& targetPoses, + const std::vector& fullBodyMask, + const std::vector& rootMask, + const std::vector& contacts, + const std::vector& endEffectorPins, + const std::vector& joint_parents_vec, + const std::vector& defaultPose, + float contactThreshold, + float root_margin, + bool has_double_ankle_joints +) +{ + + // Calculate some weights so we can preserve velocities more strongly on frames where + // the root velocity is low + const uint32_t N = poses.size(); + Eigen::VectorXd velocity_weights(N); + for (uint32_t frame = 1; frame < N; ++frame) + { + // work out xz velocity for this frame: + float xdiff = poses[frame][0].GetTranslation()[0] - poses[frame - 1][0].GetTranslation()[0]; + float zdiff = poses[frame][0].GetTranslation()[2] - poses[frame - 1][0].GetTranslation()[2]; + + // find velocity magnitude, divided by a typical walking speed: + float v_mag = sqrtf(xdiff*xdiff + zdiff*zdiff) / 0.05f; + + // weight lower velocities higher so that the corrector doesn't make the character drift around + // when it's supposed to stand still: + v_mag = std::max(v_mag, 1.0f/1000.0f); + velocity_weights(frame) = 1.0f / v_mag; + } + velocity_weights[0] = velocity_weights[1]; + + // Correct root y coordinates. + // This will warp the root y coordinates in "poses" so they match the root y coordinates + // in "targetPoses", on frames where the root y coordinates are constrained, ie the frames + // where fullBodyMask = 1. + // In addition to this, it preserves the root y coordinates in "pose" on frames where foot + // contacts are active, to avoid mushiness when characters are jumping. + CorrectHipsY( + poses, + targetPoses, + fullBodyMask, + contacts, + contactThreshold + ); + + // Correct root xz coordinates: + // This will warp the root xz coordinates in "poses" so they match the xz coordinates + // in "targetPoses" on frames where fullBodyMask = 1, and warp them so they're within + // "root_margin" units of targetPoses on frames where rootMask = 1. + CorrectHipsXZ( + poses, + targetPoses, + fullBodyMask, + rootMask, + endEffectorPins, + velocity_weights, + root_margin + ); + + // Correct joint rotations by warping the rotations so they match targetPoses on frames + // where fullBodyMask = 1: + CorrectJointRotations( + poses, + targetPoses, + fullBodyMask, + velocity_weights + ); + + // Apply IK for end effector pins + DoEffectorIK( + poses, + targetPoses, + fullBodyMask, + endEffectorPins, + joint_parents_vec, + defaultPose + ); + + // Apply IK to stabilize limbs on contacts + DoContactIK( + poses, + fullBodyMask, + contacts, + endEffectorPins, + joint_parents_vec, + defaultPose, + contactThreshold, + has_double_ankle_joints + ); + // std::cout << "Running post processing." << std::endl; +} diff --git a/MotionCorrection/src/cpp/AnimProcessing/Utility.h b/MotionCorrection/src/cpp/AnimProcessing/Utility.h new file mode 100644 index 0000000000000000000000000000000000000000..9eefbb18c93d2b83c20cf2e66f4e2ab24332c289 --- /dev/null +++ b/MotionCorrection/src/cpp/AnimProcessing/Utility.h @@ -0,0 +1,54 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Math/Transform.h" + +#include +#include + +namespace Animation +{ + enum IKType { + kOneBone, + kTwoBone + }; + + Math::Transform JointLocalToGlobal( + const std::vector& joint_parents_vec, + int32_t index, + const std::vector& localPose, + const Math::Transform& rootTx = Math::Transform::Identity + ); + + struct ContactInfo { + // index IK contact joint: + int jointIndex; + // mask indicating which frames are in contact: + std::vector contactMask; + // contact type: + IKType contactType = kTwoBone; + + // Extra info for TwoBoneIK + Math::Vector hintOffset = Math::Vector::Zero; + + float minHeight = 0.0f; + }; + + void CorrectMotion( + std::vector< std::vector >& poses, + const std::vector< std::vector >& targetPoses, + const std::vector& mask, + const std::vector& rootMask, + const std::vector& contacts, + const std::vector& endEffectorPins, + const std::vector& joint_parents_vec, + const std::vector& defaultPose, + float contactThreshold, + float root_margin, + bool has_double_ankle_joints + ); +} diff --git a/MotionCorrection/src/cpp/BindingsPython.cpp b/MotionCorrection/src/cpp/BindingsPython.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fa0993e26fc239d00a3e3871453c3d1cce84ed78 --- /dev/null +++ b/MotionCorrection/src/cpp/BindingsPython.cpp @@ -0,0 +1,364 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "AnimProcessing/Utility.h" + +#ifdef _WIN32 +#pragma warning(push) +#pragma warning(disable : 4623 4191 4686 4868 5219 4191 4355) +#endif +#include +#include +#include +#ifdef _WIN32 +#pragma warning(pop) +#endif + +namespace py = pybind11; + +float strip_nan_inf(float x) noexcept +{ + if (std::isnan(x)) return 0; + if (std::isinf(x)) return 0; + return x; +} + +void correct_motion( + py::array_t &rootTranslations, + py::array_t &jointRotations, + const py::array_t& rootTranslationsTarget, + const py::array_t& jointRotationsTarget, + const py::array_t& fullPoseMask, + const py::array_t& leftHandMask, + const py::array_t& rightHandMask, + const py::array_t& leftFootMask, + const py::array_t& rightFootMask, + const py::array_t& rootMask, + const py::array_t& contacts, + const py::list& joint_parents, + const py::list& joint_ref_translations, + const py::list& joint_ref_rotations, + int left_hand_idx, + int right_hand_idx, + int left_foot_idx, + int right_foot_idx, + float contact_threshold, + float root_margin, + bool has_double_ankle_joints +) +{ + if(joint_parents.size() != joint_ref_translations.size()) + { + throw std::runtime_error("correct_motion python bindings: joint_parents and joint_ref_translations must have the same size"); + } + if(joint_parents.size() != joint_ref_rotations.size()) + { + throw std::runtime_error("correct_motion python bindings: joint_parents and joint_ref_rotations must have the same size"); + } + if(left_hand_idx < 0 || right_hand_idx < 0 || left_foot_idx < 0 || right_foot_idx < 0) + { + throw std::runtime_error("correct_motion python bindings: left_hand_idx, right_hand_idx, left_foot_idx, and right_foot_idx must be non-negative"); + } + if(left_hand_idx >= joint_parents.size() || right_hand_idx >= joint_parents.size() || left_foot_idx >= joint_parents.size() || right_foot_idx >= joint_parents.size()) + { + throw std::runtime_error("correct_motion python bindings: left_hand_idx, right_hand_idx, left_foot_idx, and right_foot_idx must be less than the number of joints"); + } + + std::vector defaultPose(joint_parents.size()); + for (size_t i = 0; i < joint_ref_translations.size(); ++i) + { + if (!py::isinstance(joint_ref_translations[i])) + { + throw std::runtime_error("correct_motion python bindings: Expected joint_ref_translations to be a list of lists"); + } + py::list inner_list = joint_ref_translations[i].cast(); + if (inner_list.size() != 3) { + throw std::runtime_error("correct_motion python bindings: Expected joint_ref_translations to be a list of lists, length 3"); + } + + if ( + !py::isinstance(inner_list[0]) || + !py::isinstance(inner_list[1]) || + !py::isinstance(inner_list[2]) + ) + { + throw std::runtime_error("correct_motion python bindings: Expected joint_ref_translations to be a list of lists, length 3, float values"); + } + + + if (!py::isinstance(joint_ref_rotations[i])) + { + throw std::runtime_error("correct_motion python bindings: Expected joint_ref_rotations to be a list of lists"); + } + py::list inner_list_rot = joint_ref_rotations[i].cast(); + if (inner_list_rot.size() != 4) { + throw std::runtime_error("correct_motion python bindings: Expected joint_ref_rotations to be a list of lists, length 4"); + } + + if ( + !py::isinstance(inner_list_rot[0]) || + !py::isinstance(inner_list_rot[1]) || + !py::isinstance(inner_list_rot[2]) || + !py::isinstance(inner_list_rot[3]) + ) + { + throw std::runtime_error("correct_motion python bindings: Expected joint_ref_rotations to be a list of lists, length 4, float values"); + } + + defaultPose[i].SetTranslation(Math::Vector( + inner_list[0].cast(), + inner_list[1].cast(), + inner_list[2].cast())); + defaultPose[i].SetRotation(Math::Quaternion( + inner_list_rot[0].cast(), + inner_list_rot[1].cast(), + inner_list_rot[2].cast(), + inner_list_rot[3].cast() + )); + } + + std::vector joint_parents_vec(joint_parents.size()); + for (size_t i = 0; i < joint_parents.size(); ++i) + { + if (!py::isinstance(joint_parents[i])) + { + throw std::runtime_error("correct_motion python bindings: Expected joint_parents to be a list of ints"); + } + joint_parents_vec[i] = joint_parents[i].cast(); + if (joint_parents_vec[i] >= (int)joint_parents.size()) + { + throw std::runtime_error("correct_motion python bindings: joint_parents must be a list of ints, and all values must be less than the number of joints"); + } + } + + size_t num_joints = defaultPose.size(); + size_t gen_length = fullPoseMask.size(); + + if( + leftHandMask.size() != (int)gen_length || + rightHandMask.size() != (int)gen_length || + leftFootMask.size() != (int)gen_length || + rightFootMask.size() != (int)gen_length || + rootMask.size() != (int)gen_length + ) + { + throw std::runtime_error("correct_motion python bindings: all masks must have the same size"); + } + + if(rootTranslations.size() != 3 * (int)gen_length) + { + throw std::runtime_error("correct_motion python bindings: rootTranslations has the wrong size"); + } + if(jointRotations.size() != 4 * (int)num_joints * (int)gen_length) + { + throw std::runtime_error("correct_motion python bindings: jointRotations has the wrong size"); + } + + if(rootTranslationsTarget.size() != 3 * (int)gen_length) + { + throw std::runtime_error("correct_motion python bindings: rootTranslationsTarget has the wrong size"); + } + if(jointRotationsTarget.size() != 4 * (int)num_joints * (int)gen_length) + { + throw std::runtime_error("correct_motion python bindings: jointRotationsTarget has the wrong size"); + } + + std::vector endEffectorPins(4); + endEffectorPins[0].jointIndex = left_hand_idx; + endEffectorPins[0].hintOffset = Math::Vector(0.0f, 0.0f, -0.1f); + + endEffectorPins[1].jointIndex = right_hand_idx; + endEffectorPins[1].hintOffset = Math::Vector(0.0f, 0.0f, -0.1f); + + endEffectorPins[2].jointIndex = left_foot_idx; + endEffectorPins[2].hintOffset = Math::Vector(0.0f, 0.0f, 0.1f); + + endEffectorPins[3].jointIndex = right_foot_idx; + endEffectorPins[3].hintOffset = Math::Vector(0.0f, 0.0f, 0.1f); + + endEffectorPins[0].contactMask.reserve(gen_length); + endEffectorPins[1].contactMask.reserve(gen_length); + endEffectorPins[2].contactMask.reserve(gen_length); + endEffectorPins[3].contactMask.reserve(gen_length); + for(size_t i = 0; i < gen_length; ++i) + { + endEffectorPins[0].contactMask.push_back((1.0f - fullPoseMask.at(i)) * leftHandMask.at(i)); + endEffectorPins[1].contactMask.push_back((1.0f - fullPoseMask.at(i)) * rightHandMask.at(i)); + endEffectorPins[2].contactMask.push_back((1.0f - fullPoseMask.at(i)) * leftFootMask.at(i)); + endEffectorPins[3].contactMask.push_back((1.0f - fullPoseMask.at(i)) * rightFootMask.at(i)); + } + + std::vector contactInfo(2); + + auto footTranslation = Animation::JointLocalToGlobal( + joint_parents_vec, + right_foot_idx, + defaultPose + ).GetTranslation(); + + contactInfo[0].jointIndex = right_foot_idx; + contactInfo[0].hintOffset = Math::Vector(0.0f, 0.0f, 0.1f); + contactInfo[0].minHeight = footTranslation.GetY(); + + footTranslation = Animation::JointLocalToGlobal( + joint_parents_vec, + left_foot_idx, + defaultPose + ).GetTranslation(); + + contactInfo[1].jointIndex = left_foot_idx; + contactInfo[1].hintOffset = Math::Vector(0.0f, 0.0f, 0.1f); + contactInfo[1].minHeight = footTranslation.GetY(); + + auto& rContacts = contactInfo[0].contactMask; + auto& lContacts = contactInfo[1].contactMask; + + rContacts.resize(fullPoseMask.size()); + lContacts.resize(fullPoseMask.size()); + for (int i = 0; i < fullPoseMask.size(); ++i) + { + // don't flag it as a contact if it's been masked: + rContacts[i] = rightFootMask.at(i) ? 0 : contacts.at(4 * i + 2); + lContacts[i] = leftFootMask.at(i) ? 0 : contacts.at(4 * i + 0); + + // Flag the heel as a contact if the toe is a contact: + rContacts[i] = std::min((rightFootMask.at(i) ? 0 : contacts.at(4 * i + 3)) + rContacts[i], 1.0f); + lContacts[i] = std::min((leftFootMask.at(i) ? 0 : contacts.at(4 * i + 1)) + lContacts[i], 1.0f); + } + + int left_toe_idx = -1; + int right_toe_idx = -1; + for(int i = 0; i < num_joints; ++i) + { + if(joint_parents_vec[i] == left_foot_idx) + { + left_toe_idx = i; + } + if(joint_parents_vec[i] == right_foot_idx) + { + right_toe_idx = i; + } + } + + if(left_toe_idx != -1 && right_toe_idx != -1) + { + auto toeTranslation = Animation::JointLocalToGlobal( + joint_parents_vec, + right_toe_idx, + defaultPose + ).GetTranslation(); + + contactInfo.resize(4); + contactInfo[2].jointIndex = right_toe_idx; + contactInfo[2].contactType = Animation::kOneBone; + contactInfo[2].minHeight = toeTranslation.GetY(); + + contactInfo[3].jointIndex = left_toe_idx; + contactInfo[3].contactType = Animation::kOneBone; + contactInfo[3].minHeight = toeTranslation.GetY(); + + auto& rToeContacts = contactInfo[2].contactMask; + auto& lToeContacts = contactInfo[3].contactMask; + + // fill up the ankle contacts: + rToeContacts.resize(fullPoseMask.size()); + lToeContacts.resize(fullPoseMask.size()); + + for (int i = 0; i < fullPoseMask.size(); ++i) + { + // don't flag it as a contact if it's been masked: + rToeContacts[i] = rightFootMask.at(i) ? 0 : contacts.at(4 * i + 3); + lToeContacts[i] = leftFootMask.at(i) ? 0 : contacts.at(4 * i + 1); + } + } + + + auto setTransforms = [gen_length, num_joints]( + std::vector< std::vector > &poses, + const py::array_t &rootTranslations, + const py::array_t &jointRotations + ) + { + for (size_t f = 0; f < gen_length; ++f) + { + poses[f][0].SetTranslation({ + strip_nan_inf(rootTranslations.at(3*f+0)), + strip_nan_inf(rootTranslations.at(3*f+1)), + strip_nan_inf(rootTranslations.at(3*f+2)) + }); + } + + for (size_t f = 0; f < gen_length; ++f) + { + for (size_t j = 0; j < num_joints; ++j) + { + // x y z w order: + Math::Quaternion q( + strip_nan_inf(jointRotations.at(4 * (num_joints * f + j) + 1)), + strip_nan_inf(jointRotations.at(4 * (num_joints * f + j) + 2)), + strip_nan_inf(jointRotations.at(4 * (num_joints * f + j) + 3)), + strip_nan_inf(jointRotations.at(4 * (num_joints * f + j) + 0)) + ); + q.Normalize(); + poses[f][j].SetRotation(q); + } + } + }; + + std::vector< std::vector > posesFixed(gen_length, defaultPose); + setTransforms(posesFixed, rootTranslations, jointRotations); + + std::vector< std::vector > posesTarget(gen_length, defaultPose); + setTransforms(posesTarget, rootTranslationsTarget, jointRotationsTarget); + + std::vector fullPoseMask_vec; + std::vector rootMask_vec; + for (size_t f = 0; f < gen_length; ++f) + { + fullPoseMask_vec.push_back(fullPoseMask.at(f)); + rootMask_vec.push_back(rootMask.at(f)); + } + + Animation::CorrectMotion( + posesFixed, + posesTarget, + fullPoseMask_vec, + rootMask_vec, + contactInfo, + endEffectorPins, + joint_parents_vec, + defaultPose, + contact_threshold, + root_margin, + has_double_ankle_joints + ); + + for (size_t f = 0; f < gen_length; ++f) + { + auto t = posesFixed[f][0].GetTranslation(); + rootTranslations.mutable_at(3*f+0) = t.GetX(); + rootTranslations.mutable_at(3*f+1) = t.GetY(); + rootTranslations.mutable_at(3*f+2) = t.GetZ(); + } + + for (size_t f = 0; f < gen_length; ++f) + { + for (size_t j = 0; j < num_joints; ++j) + { + auto q = posesFixed[f][j].GetRotation(); + // w x y z order + jointRotations.mutable_at(4 * (num_joints * f + j) + 0) = ((float*)&q)[3]; + jointRotations.mutable_at(4 * (num_joints * f + j) + 1) = ((float*)&q)[0]; + jointRotations.mutable_at(4 * (num_joints * f + j) + 2) = ((float*)&q)[1]; + jointRotations.mutable_at(4 * (num_joints * f + j) + 3) = ((float*)&q)[2]; + } + } + +} + +PYBIND11_MODULE(_motion_correction, m) { + m.doc() = "Motion Correction Python bindings"; + m.def("correct_motion", &correct_motion); +} diff --git a/MotionCorrection/src/cpp/Compiler.h b/MotionCorrection/src/cpp/Compiler.h new file mode 100644 index 0000000000000000000000000000000000000000..851f934400b241baccb0d2b81a73cc1a228c7b28 --- /dev/null +++ b/MotionCorrection/src/cpp/Compiler.h @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +// Compiler specific defines + +// Finds the compiler type and version. +#if defined(__clang__) +# define COMPILER_CLANG +#elif defined(__GNUC__) // Check after Clang, as Clang defines this too +# define COMPILER_GNUC +#elif defined(_MSC_VER) // Check after Clang, since we could be building with either within VS +# define COMPILER_MSVC +#else +# pragma error "Unknown compiler. " +#endif + +#if defined(COMPILER_MSVC) + #define FORCE_INLINE __forceinline +#elif defined(COMPILER_GNUC) + #define FORCE_INLINE inline __attribute__((always_inline)) +#endif diff --git a/MotionCorrection/src/cpp/Debug.h b/MotionCorrection/src/cpp/Debug.h new file mode 100644 index 0000000000000000000000000000000000000000..fc3499d1ba20d23aed651a54641906f68f99d859 --- /dev/null +++ b/MotionCorrection/src/cpp/Debug.h @@ -0,0 +1,13 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Platform.h" + +#define ASSERT( cond ) do { if( !(cond) ) { DEBUG_BREAK(); } } while( 0 ) +#define HALT() { DEBUG_BREAK(); } +#define UNIMPLEMENTED_FUNCTION() { DEBUG_BREAK(); } +#define UNREACHABLE_CODE() { DEBUG_BREAK(); } diff --git a/MotionCorrection/src/cpp/Math/Constants.h b/MotionCorrection/src/cpp/Math/Constants.h new file mode 100644 index 0000000000000000000000000000000000000000..4717cd5ae04977f97bc4e6de5dd4796cb5b804f6 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Constants.h @@ -0,0 +1,32 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +// Mathematical constants + +namespace Math +{ + static constexpr float const Epsilon = 1.0e-06f; + static constexpr float const LargeEpsilon = 1.0e-04f; + static constexpr float const HugeEpsilon = 1.0e-02f; + static constexpr float const Pi = 3.141592654f; + static constexpr float const TwoPi = 6.283185307f; + static constexpr float const OneDivPi = 0.318309886f; + static constexpr float const OneDivTwoPi = 0.159154943f; + static constexpr float const PiDivTwo = 1.570796327f; + static constexpr float const PiDivFour = 0.785398163f; + + static constexpr float const SqrtTwo = 1.4142135623730950488016887242097f; + static constexpr float const OneDivSqrtTwo = 1.0f / SqrtTwo; + + static constexpr float const DegreesToRadians = 0.0174532925f; + static constexpr float const RadiansToDegrees = 57.2957795f; + + static constexpr float const Infinity = std::numeric_limits::infinity(); + static constexpr float const QNaN = std::numeric_limits::quiet_NaN(); +} diff --git a/MotionCorrection/src/cpp/Math/Matrix.cpp b/MotionCorrection/src/cpp/Math/Matrix.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7c069271d1445315972a7255a67af3eef3547a74 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Matrix.cpp @@ -0,0 +1,305 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "Matrix.h" + +#include + +using namespace Math; + +namespace +{ + static bool CheckForZeroScaleInRow(float scale, const Vector& row) + { + float const absScale = Math::Abs(scale); + + for (int i = 0; i < 3; i++) + { + if (absScale < 1 && Math::Abs(row[i]) >= FLT_MAX * absScale) + { + return false; + } + } + + return true; + } + + static bool ExtractAndRemoveScalingAndShear(Matrix& matrix, Vector& scale, Vector& shear) + { + scale = Vector::Zero; + shear = Vector::Zero; + + Float3 scaleValues = Float3::Zero; + Float3 shearValues = Float3::Zero; + + // This implementation follows the technique described in the paper by + // Spencer W. Thomas in the Graphics Gems II article: "Decomposing a + // Matrix into Simple Transformations", p. 320. + + Vector row[3]; + row[0] = Vector(matrix[0][0], matrix[0][1], matrix[0][2]); + row[1] = Vector(matrix[1][0], matrix[1][1], matrix[1][2]); + row[2] = Vector(matrix[2][0], matrix[2][1], matrix[2][2]); + + float maxVal = 0; + for (int i = 0; i < 3; i++) + { + for (int j = 0; j < 3; j++) + { + if (Math::Abs(row[i][j]) > maxVal) + { + maxVal = Math::Abs(row[i][j]); + } + } + } + + // We normalize the 3x3 matrix here. + // It was noticed that this can improve numerical stability significantly, + // especially when many of the upper 3x3 matrix's coefficients are very + // close to zero; we correct for this step at the end by multiplying the + // scaling factors by maxVal at the end (shear and rotation are not + // affected by the normalization). + + if (maxVal != 0) + { + for (int i = 0; i < 3; i++) + { + if (!CheckForZeroScaleInRow(maxVal, row[i])) + { + return false; + } + else + { + row[i] /= maxVal; + } + } + } + + // Compute X scale factor. + scaleValues.m_x = row[0].Length3().ToFloat(); + if (!CheckForZeroScaleInRow(scaleValues.m_x, row[0])) + { + return false; + } + + // Normalize first row. + row[0] /= scaleValues.m_x; + + // An XY shear factor will shear the X coord. as the Y coord. changes. + // There are 6 combinations (XY, XZ, YZ, YX, ZX, ZY), although we only + // extract the first 3 because we can effect the last 3 by shearing in + // XY, XZ, YZ combined rotations and scales. + // + // shear matrix < 1, YX, ZX, 0, + // XY, 1, ZY, 0, + // XZ, YZ, 1, 0, + // 0, 0, 0, 1 > + + // Compute XY shear factor and make 2nd row orthogonal to 1st. + shearValues[0] = Vector::Dot3(row[0], row[1]).ToFloat(); + row[1] -= row[0] * shearValues[0]; + + // Now, compute Y scale. + scaleValues.m_y = row[1].Length3().ToFloat(); + if (!CheckForZeroScaleInRow(scaleValues.m_y, row[1])) + { + return false; + } + + // Normalize 2nd row and correct the XY shear factor for Y scaling. + row[1] /= scaleValues.m_y; + shearValues[0] /= scaleValues.m_y; + + // Compute XZ and YZ shears, orthogonalize 3rd row. + shearValues[1] = Vector::Dot3(row[0], row[2]).ToFloat(); + row[2] -= row[0] * shearValues[1]; + shearValues[2] = Vector::Dot3(row[1], row[2]).ToFloat(); + row[2] -= row[1] * shearValues[2]; + + // Next, get Z scale. + scaleValues.m_z = row[2].Length3().ToFloat(); + if (!CheckForZeroScaleInRow(scaleValues.m_z, row[2])) + { + return false; + } + + // Normalize 3rd row and correct the XZ and YZ shear factors for Z scaling. + row[2] /= scaleValues.m_z; + shearValues[1] /= scaleValues.m_z; + shearValues[2] /= scaleValues.m_z; + + // At this point, the upper 3x3 matrix in mat is orthonormal. + // Check for a coordinate system flip. If the determinant + // is less than zero, then negate the matrix and the scaling factors. + if (Vector::Dot3(row[0], Vector::Cross3(row[1], row[2])).ToFloat() < 0) + { + for (int i = 0; i < 3; i++) + { + scaleValues[i] *= -1; + row[i] *= -1; + } + } + + // Copy over the orthonormal rows into the returned matrix. + // The upper 3x3 matrix in mat is now a rotation matrix. + for (int i = 0; i < 3; i++) + { + matrix[i].SetX(row[i][0]); + matrix[i].SetY(row[i][1]); + matrix[i].SetZ(row[i][2]); + } + + // Correct the scaling factors for the normalization step that we + // performed above; shear and rotation are not affected by the + // normalization. + scaleValues *= maxVal; + + scale = Vector(scaleValues); + shear = Vector(shearValues); + + return true; + } +} + +namespace Math +{ + Matrix const Matrix::Identity(1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1); + + Matrix::Matrix(float v00, float v01, float v02, float v03, float v10, float v11, float v12, float v13, float v20, float v21, float v22, float v23, float v30, float v31, float v32, float v33) + { + m_rows[0] = Vector(v00, v01, v02, v03); + m_rows[1] = Vector(v10, v11, v12, v13); + m_rows[2] = Vector(v20, v21, v22, v23); + m_rows[3] = Vector(v30, v31, v32, v33); + } + + Matrix::Matrix(float values[16]) + { + m_rows[0] = Vector(values[0], values[1], values[2], values[3]); + m_rows[1] = Vector(values[4], values[5], values[6], values[7]); + m_rows[2] = Vector(values[8], values[9], values[10], values[11]); + m_rows[3] = Vector(values[12], values[13], values[14], values[15]); + } + + Matrix::Matrix(const Vector& xAxis, const Vector& yAxis, const Vector& zAxis) + { + m_rows[0] = xAxis; + m_rows[1] = yAxis; + m_rows[2] = zAxis; + m_rows[3] = Vector::UnitW; + } + + Matrix::Matrix(const Vector& xAxis, const Vector& yAxis, const Vector& zAxis, const Vector& translation) + { + m_rows[0] = xAxis; + m_rows[1] = yAxis; + m_rows[2] = zAxis; + m_rows[3] = translation.GetWithW1(); + } + + Matrix::Matrix(const EulerAngles& eulerAngles, const Vector translation) + { + float cx, cy, cz, sx, sy, sz, czsx, cxcz, sysz; + + sx = sinf((float)eulerAngles.m_x); cx = cosf((float)eulerAngles.m_x); + sy = sinf((float)eulerAngles.m_y); cy = cosf((float)eulerAngles.m_y); + sz = sinf((float)eulerAngles.m_z); cz = cosf((float)eulerAngles.m_z); + + czsx = cz * sx; + cxcz = cx * cz; + sysz = sy * sz; + + // Order is XYZ + m_values[0][0] = cy * cz; + m_values[0][1] = cy * sz; + m_values[0][2] = -sy; + m_values[1][0] = czsx * sy - cx * sz; + m_values[1][1] = cxcz + sx * sysz; + m_values[1][2] = cy * sx; + m_values[2][0] = cxcz * sy + sx * sz; + m_values[2][1] = -czsx + cx * sysz; + m_values[2][2] = cx * cy; + m_values[0][3] = 0.0f; + m_values[1][3] = 0.0f; + m_values[2][3] = 0.0f; + + // Translation + m_rows[3] = translation.GetWithW1(); + } + + EulerAngles Matrix::ToEulerAngles() const + { + EulerAngles result; + + result.m_x = Radians(Math::ATan2(m_values[1][2], m_values[2][2])); + + float const c2 = Math::Sqrt((m_values[0][0] * m_values[0][0]) + (m_values[0][1] * m_values[0][1])); + result.m_y = Radians(Math::ATan2(-m_values[0][2], c2)); + + float const s1 = Math::Sin((float)result.m_x); + float const c1 = Math::Cos((float)result.m_x); + result.m_z = Radians(Math::ATan2((s1 * m_values[2][0]) - (c1 * m_values[1][0]), (c1 * m_values[1][1]) - (s1 * m_values[2][1]))); + + return result; + } + + bool Matrix::Decompose(Quaternion& outRotation, Vector& outTranslation, Vector& outScale) const + { + Matrix copy = *this; + Vector shr = Vector::Zero; + outScale = Vector::Zero; + + // Extract and remove scale and shear from matrix + if (ExtractAndRemoveScalingAndShear(copy, outScale, shr)) + { + // Extract rotation and translation from unscaled matrix + outRotation = copy.GetRotation(); + outTranslation = copy.GetTranslation().GetWithW0(); + return true; + } + + return false; + } + + Vector Matrix::GetScale() const + { + Matrix copy = *this; + Vector scale = Vector::Zero, shear; + if (!ExtractAndRemoveScalingAndShear(copy, scale, shear)) + { + float const lengthX = m_rows[0].Length3().ToFloat(); + float const lengthY = m_rows[1].Length3().ToFloat(); + float const lengthZ = m_rows[2].Length3().ToFloat(); + scale = Vector(lengthX, lengthY, lengthZ, 0.0f); + } + + return scale; + } + + Matrix& Matrix::SetScale(const Vector& newScale) + { + Vector scale, shear; + bool result = ExtractAndRemoveScalingAndShear(*this, scale, shear); + + // Cannot set scale on matrix that contains zero-scale + ASSERT(result); + + m_rows[0] = m_rows[0] * newScale.GetSplatX(); + m_rows[1] = m_rows[1] * newScale.GetSplatY(); + m_rows[2] = m_rows[2] * newScale.GetSplatZ(); + return *this; + } + + Matrix& Matrix::RemoveScale() + { + Vector scale, shear; + bool result = ExtractAndRemoveScalingAndShear(*this, scale, shear); + + // Cannot remove zero scale from matrix + ASSERT(result); + + return *this; + } +} diff --git a/MotionCorrection/src/cpp/Math/Matrix.h b/MotionCorrection/src/cpp/Math/Matrix.h new file mode 100644 index 0000000000000000000000000000000000000000..3e0a3d62e11b625588aef489ae976a2da2b97a02 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Matrix.h @@ -0,0 +1,163 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Vector.h" +#include "Quaternion.h" + +enum class CoordinateSpace : uint8_t +{ + World, + Local, +}; + +// +// Matrices are Row-Major +// Multiplication order is right to left +// ObjectWorldTransform = LocalObjectTransform * WorldTransform +// + +namespace Math +{ + class alignas(16) Matrix + { + public: + + static Matrix const Identity; + + public: + + static Matrix FromRotation(const Quaternion& rotation); + static Matrix FromTranslation(const Vector& translation); + static Matrix FromScale(const Vector& scale); + static Matrix FromUniformScale(float uniformScale); + static Matrix FromTranslationAndScale(const Vector& translation, const Vector& scale); + static Matrix FromRotationBetweenVectors(const Vector sourceVector, const Vector targetVector); + + public: + + explicit Matrix(); + explicit Matrix(NoInit_t); + explicit Matrix(ZeroInit_t); + explicit Matrix(float v00, float v01, float v02, float v03, + float v10, float v11, float v12, float v13, + float v20, float v21, float v22, float v23, + float v30, float v31, float v32, float v33); + explicit Matrix(float values[16]); + explicit Matrix(Vector const& xAxis, Vector const& yAxis, Vector const& zAxis); + explicit Matrix(Vector const& xAxis, Vector const& yAxis, Vector const& zAxis, Vector const& translation); + + Matrix(const Vector axis, Radians angleRadians); + Matrix(const AxisAngle axisAngle); + + explicit Matrix(const Quaternion& rotation); + explicit Matrix(const Quaternion& rotation, const Vector& translation, const Vector& scale = Vector::One); + explicit Matrix(const Quaternion& rotation, const Vector& translation, float scale = 1.0f); + explicit Matrix(const EulerAngles& eulerAngles, const Vector translation = Vector::UnitW); + + EulerAngles ToEulerAngles() const; + + float* AsFloatArray(); + const float* AsFloatArray() const; + const Vector& GetRow(uint32_t row) const; + + const Vector& GetAxisX() const; + const Vector& GetAxisY() const; + const Vector& GetAxisZ() const; + + void SetAxisX(const Vector& xAxis); + void SetAxisY(const Vector& yAxis); + void SetAxisZ(const Vector& zAxis); + + Float3 GetForwardVector() const; + Float3 GetRightVector() const; + Float3 GetUpVector() const; + + Vector GetUnitAxisX() const; + Vector GetUnitAxisY() const; + Vector GetUnitAxisZ() const; + + bool IsIdentity() const; + bool IsOrthogonal() const; + bool IsOrthonormal() const; + + bool Decompose(Quaternion& outRotation, Vector& outTranslation, Vector& outScale) const; + + Matrix& Transpose(); + Matrix GetTransposed() const; + + Matrix& Invert(); + Matrix GetInverse() const; + + Vector GetDeterminant() const; + float GetDeterminantAsFloat() const; + + Vector GetTranslation() const; + const Vector& GetTranslationWithW() const; + Matrix& SetTranslation(Vector const& v); + Matrix& SetTranslation(Float3 const& v); + Matrix& SetTranslation(Float4 const& v); + + Quaternion GetRotation() const; + + Matrix& SetRotation(const Matrix& rotation); + Matrix& SetRotation(const Quaternion& rotation); + + Matrix& SetRotationMaintainingScale(const Matrix& rotation); + Matrix& SetRotationMaintainingScale(const Quaternion& rotation); + + Vector GetScale() const; + + Matrix& RemoveScale(); + Matrix& SetScale(const Vector& scale); + Matrix& SetScale(float uniformScale); + + Matrix& RemoveScaleFast(); + Matrix& SetScaleFast(const Vector& scale); + Matrix& SetScaleFast(float uniformScale); + + // + // Operators + // + + // Applies rotation and scale to a vector and returns a result with the W = 0 + Vector RotateVector(const Vector& vector) const; + + // Applies rotation and scale to a vector and returns a result with the W = 0 + Vector TransformNormal(const Vector& vector) const; + + // Applies the transformation to a given point and ensures the resulting W = 1 + Vector TransformPoint(const Vector& point) const; + + // Applies the transformation to a vector ignoring the W value. + // Same as TransformPoint with the result W left unchanged + Vector TransformVector3(const Vector& vector) const; + + // Applies the transformation to a given vector with the result W left unchanged + Vector TransformVector4(const Vector& vector) const; + + Vector& operator[](uint32_t i); + const Vector operator[](uint32_t i) const; + + Matrix operator*(const Matrix& rhs) const; + Matrix& operator*=(const Matrix& rhs); + + Matrix operator*(const Quaternion& rhs) const; + Matrix operator*=(const Quaternion& rhs); + + bool operator==(const Matrix& rhs) const; + + public: + + union + { + Vector m_rows[4]; + float m_values[4][4]; + }; + }; +} + +#include "Matrix.inl" diff --git a/MotionCorrection/src/cpp/Math/Matrix.inl b/MotionCorrection/src/cpp/Math/Matrix.inl new file mode 100644 index 0000000000000000000000000000000000000000..3c616529b8ea3c2a1c1135dc06fa76222fecf057 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Matrix.inl @@ -0,0 +1,802 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "Matrix.h" + +namespace Math +{ + inline Matrix Matrix::FromRotation(const Quaternion& rotation) + { + return Matrix(rotation); + } + + inline Matrix Matrix::FromTranslation(const Vector& translation) + { + Matrix M; + M.m_rows[0] = Vector::UnitX; + M.m_rows[1] = Vector::UnitY; + M.m_rows[2] = Vector::UnitZ; + M.m_rows[3] = translation.GetWithW1(); + return M; + } + + inline Matrix Matrix::FromScale(const Vector& scale) + { + Matrix M; + M.m_rows[0] = _mm_and_ps(scale, SIMD::g_maskX000); + M.m_rows[1] = _mm_and_ps(scale, SIMD::g_mask0Y00); + M.m_rows[2] = _mm_and_ps(scale, SIMD::g_mask00Z0); + M.m_rows[3] = Vector::UnitW; + return M; + } + + inline Matrix Matrix::FromUniformScale(float uniformScale) + { + Matrix M; + M.m_rows[0] = _mm_set_ps(0, 0, 0, uniformScale); + M.m_rows[1] = _mm_set_ps(0, 0, uniformScale, 0); + M.m_rows[2] = _mm_set_ps(0, uniformScale, 0, 0); + M.m_rows[3] = Vector::UnitW; + return M; + } + + inline Matrix Matrix::FromTranslationAndScale(const Vector& translation, const Vector& scale) + { + Matrix M; + M.m_rows[0] = _mm_and_ps(scale, SIMD::g_maskX000); + M.m_rows[1] = _mm_and_ps(scale, SIMD::g_mask0Y00); + M.m_rows[2] = _mm_and_ps(scale, SIMD::g_mask00Z0); + M.m_rows[3] = translation.GetWithW1(); + return M; + } + + inline Matrix Matrix::FromRotationBetweenVectors(Vector const sourceVector, Vector const targetVector) + { + return Matrix(Quaternion::FromRotationBetweenNormalizedVectors(sourceVector, targetVector)); + } + + inline Matrix::Matrix() + { + memcpy(this, &Matrix::Identity, sizeof(Matrix)); + } + + inline Matrix::Matrix(NoInit_t) + { + } + + inline Matrix::Matrix(ZeroInit_t) + { + memset(this, 0, sizeof(Matrix)); + } + + inline Matrix::Matrix(const Vector axis, Radians angleRadians) + { + Vector normal = axis.GetNormalized3(); + + Vector C0, C1; + Vector::SinCos(C0, C1, Vector((float)angleRadians)); + Vector C2 = Vector::One - C1; + + __m128 N0 = _mm_shuffle_ps(normal, normal, _MM_SHUFFLE(3, 0, 2, 1)); + __m128 N1 = _mm_shuffle_ps(normal, normal, _MM_SHUFFLE(3, 1, 0, 2)); + + __m128 V0 = _mm_mul_ps(C2, N0); + V0 = _mm_mul_ps(V0, N1); + + __m128 R0 = _mm_mul_ps(C2, normal); + R0 = _mm_mul_ps(R0, normal); + R0 = _mm_add_ps(R0, C1); + + __m128 R1 = _mm_mul_ps(C0, normal); + R1 = _mm_add_ps(R1, V0); + __m128 R2 = _mm_mul_ps(C0, normal); + R2 = _mm_sub_ps(V0, R2); + + V0 = _mm_and_ps(R0, SIMD::g_maskXYZ0); + __m128 V1 = _mm_shuffle_ps(R1, R2, _MM_SHUFFLE(2, 1, 2, 0)); + V1 = _mm_shuffle_ps(V1, V1, _MM_SHUFFLE(0, 3, 2, 1)); + __m128 V2 = _mm_shuffle_ps(R1, R2, _MM_SHUFFLE(0, 0, 1, 1)); + V2 = _mm_shuffle_ps(V2, V2, _MM_SHUFFLE(2, 0, 2, 0)); + + R2 = _mm_shuffle_ps(V0, V1, _MM_SHUFFLE(1, 0, 3, 0)); + R2 = _mm_shuffle_ps(R2, R2, _MM_SHUFFLE(1, 3, 2, 0)); + + m_rows[0] = R2; + + R2 = _mm_shuffle_ps(V0, V1, _MM_SHUFFLE(3, 2, 3, 1)); + R2 = _mm_shuffle_ps(R2, R2, _MM_SHUFFLE(1, 3, 0, 2)); + m_rows[1] = R2; + + V2 = _mm_shuffle_ps(V2, V0, _MM_SHUFFLE(3, 2, 1, 0)); + m_rows[2] = V2; + m_rows[3] = Vector::UnitW; + } + + inline Matrix::Matrix(const AxisAngle axisAngle) + : Matrix(Vector(axisAngle.m_axis), axisAngle.m_angle) + { + } + + inline Matrix::Matrix(const Quaternion& rotation) + { + SetRotation(rotation); + m_rows[3] = Vector::UnitW; + } + + inline Matrix::Matrix(const Quaternion& rotation, const Vector& translation, const Vector& scale) + { + SetRotation(rotation); + m_rows[0] = m_rows[0] * scale.GetSplatX(); + m_rows[1] = m_rows[1] * scale.GetSplatY(); + m_rows[2] = m_rows[2] * scale.GetSplatZ(); + m_rows[3] = translation.GetWithW1(); + } + + inline Matrix::Matrix(const Quaternion& rotation, const Vector& translation, float scale) + : Matrix(rotation, translation, Vector(scale)) + { + } + + inline float* Matrix::AsFloatArray() + { + return &m_values[0][0]; + } + + inline const float* Matrix::AsFloatArray() const + { + return &m_values[0][0]; + } + + inline const Vector& Matrix::GetRow(uint32_t row) const + { + return m_rows[row]; + } + + inline const Vector& Matrix::GetAxisX() const + { + return m_rows[0]; + } + + inline const Vector& Matrix::GetAxisY() const + { + return m_rows[1]; + } + + inline const Vector& Matrix::GetAxisZ() const + { + return m_rows[2]; + } + + inline void Matrix::SetAxisX(const Vector& xAxis) + { + m_rows[0] = xAxis; + } + + inline void Matrix::SetAxisY(const Vector& yAxis) + { + m_rows[1] = yAxis; + } + + inline void Matrix::SetAxisZ(const Vector& zAxis) + { + m_rows[2] = zAxis; + } + + + inline Float3 Matrix::GetForwardVector() const + { + return GetAxisZ(); + } + + inline Float3 Matrix::GetRightVector() const + { + return GetAxisX(); + } + + inline Float3 Matrix::GetUpVector() const + { + return GetAxisY(); + } + + inline Vector Matrix::GetUnitAxisX() const + { + return m_rows[0].GetNormalized3(); + } + + inline Vector Matrix::GetUnitAxisY() const + { + return m_rows[1].GetNormalized3(); + } + + inline Vector Matrix::GetUnitAxisZ() const + { + return m_rows[2].GetNormalized3(); + } + + inline bool Matrix::IsIdentity() const + { + __m128 vTemp1 = _mm_cmpeq_ps(m_rows[0], Vector::UnitX); + __m128 vTemp2 = _mm_cmpeq_ps(m_rows[1], Vector::UnitY); + __m128 vTemp3 = _mm_cmpeq_ps(m_rows[2], Vector::UnitZ); + __m128 vTemp4 = _mm_cmpeq_ps(m_rows[3], Vector::UnitW); + vTemp1 = _mm_and_ps(vTemp1, vTemp2); + vTemp3 = _mm_and_ps(vTemp3, vTemp4); + vTemp1 = _mm_and_ps(vTemp1, vTemp3); + return (_mm_movemask_ps(vTemp1) == 0x0f); + } + + inline bool Matrix::IsOrthogonal() const + { + Matrix const transpose = GetTransposed(); + Matrix result = *this * transpose; + return result.IsIdentity(); + } + + inline bool Matrix::IsOrthonormal() const + { + static const Vector three(3); + auto dotCheck = Vector::Dot3(m_rows[0], m_rows[1]) + Vector::Dot3(m_rows[0], m_rows[2]) + Vector::Dot3(m_rows[1], m_rows[2]); + auto magnitudeCheck = m_rows[0].LengthSquared3() + m_rows[1].LengthSquared3() + m_rows[2].LengthSquared3(); + auto result = dotCheck + magnitudeCheck; + return result.IsNearEqual3(three); + } + + inline Matrix& Matrix::Transpose() + { + __m128 vTemp1 = _mm_shuffle_ps(m_rows[0], m_rows[1], _MM_SHUFFLE(1, 0, 1, 0)); + __m128 vTemp3 = _mm_shuffle_ps(m_rows[0], m_rows[1], _MM_SHUFFLE(3, 2, 3, 2)); + __m128 vTemp2 = _mm_shuffle_ps(m_rows[2], m_rows[3], _MM_SHUFFLE(1, 0, 1, 0)); + __m128 vTemp4 = _mm_shuffle_ps(m_rows[2], m_rows[3], _MM_SHUFFLE(3, 2, 3, 2)); + m_rows[0] = _mm_shuffle_ps(vTemp1, vTemp2, _MM_SHUFFLE(2, 0, 2, 0)); + m_rows[1] = _mm_shuffle_ps(vTemp1, vTemp2, _MM_SHUFFLE(3, 1, 3, 1)); + m_rows[2] = _mm_shuffle_ps(vTemp3, vTemp4, _MM_SHUFFLE(2, 0, 2, 0)); + m_rows[3] = _mm_shuffle_ps(vTemp3, vTemp4, _MM_SHUFFLE(3, 1, 3, 1)); + return *this; + } + + inline Matrix Matrix::GetTransposed() const + { + Matrix m = *this; + m.Transpose(); + return m; + } + + inline Matrix& Matrix::Invert() + { + Matrix MT = GetTransposed(); + __m128 V00 = _mm_shuffle_ps(MT.m_rows[2], MT.m_rows[2], _MM_SHUFFLE(1, 1, 0, 0)); + __m128 V10 = _mm_shuffle_ps(MT.m_rows[3], MT.m_rows[3], _MM_SHUFFLE(3, 2, 3, 2)); + __m128 V01 = _mm_shuffle_ps(MT.m_rows[0], MT.m_rows[0], _MM_SHUFFLE(1, 1, 0, 0)); + __m128 V11 = _mm_shuffle_ps(MT.m_rows[1], MT.m_rows[1], _MM_SHUFFLE(3, 2, 3, 2)); + __m128 V02 = _mm_shuffle_ps(MT.m_rows[2], MT.m_rows[0], _MM_SHUFFLE(2, 0, 2, 0)); + __m128 V12 = _mm_shuffle_ps(MT.m_rows[3], MT.m_rows[1], _MM_SHUFFLE(3, 1, 3, 1)); + + __m128 D0 = _mm_mul_ps(V00, V10); + __m128 D1 = _mm_mul_ps(V01, V11); + __m128 D2 = _mm_mul_ps(V02, V12); + + V00 = _mm_shuffle_ps(MT.m_rows[2], MT.m_rows[2], _MM_SHUFFLE(3, 2, 3, 2)); + V10 = _mm_shuffle_ps(MT.m_rows[3], MT.m_rows[3], _MM_SHUFFLE(1, 1, 0, 0)); + V01 = _mm_shuffle_ps(MT.m_rows[0], MT.m_rows[0], _MM_SHUFFLE(3, 2, 3, 2)); + V11 = _mm_shuffle_ps(MT.m_rows[1], MT.m_rows[1], _MM_SHUFFLE(1, 1, 0, 0)); + V02 = _mm_shuffle_ps(MT.m_rows[2], MT.m_rows[0], _MM_SHUFFLE(3, 1, 3, 1)); + V12 = _mm_shuffle_ps(MT.m_rows[3], MT.m_rows[1], _MM_SHUFFLE(2, 0, 2, 0)); + + V00 = _mm_mul_ps(V00, V10); + V01 = _mm_mul_ps(V01, V11); + V02 = _mm_mul_ps(V02, V12); + D0 = _mm_sub_ps(D0, V00); + D1 = _mm_sub_ps(D1, V01); + D2 = _mm_sub_ps(D2, V02); + // V11 = D0Y,D0W,D2Y,D2Y + V11 = _mm_shuffle_ps(D0, D2, _MM_SHUFFLE(1, 1, 3, 1)); + V00 = _mm_shuffle_ps(MT.m_rows[1], MT.m_rows[1], _MM_SHUFFLE(1, 0, 2, 1)); + V10 = _mm_shuffle_ps(V11, D0, _MM_SHUFFLE(0, 3, 0, 2)); + V01 = _mm_shuffle_ps(MT.m_rows[0], MT.m_rows[0], _MM_SHUFFLE(0, 1, 0, 2)); + V11 = _mm_shuffle_ps(V11, D0, _MM_SHUFFLE(2, 1, 2, 1)); + // V13 = D1Y,D1W,D2W,D2W + __m128 V13 = _mm_shuffle_ps(D1, D2, _MM_SHUFFLE(3, 3, 3, 1)); + V02 = _mm_shuffle_ps(MT.m_rows[3], MT.m_rows[3], _MM_SHUFFLE(1, 0, 2, 1)); + V12 = _mm_shuffle_ps(V13, D1, _MM_SHUFFLE(0, 3, 0, 2)); + __m128 V03 = _mm_shuffle_ps(MT.m_rows[2], MT.m_rows[2], _MM_SHUFFLE(0, 1, 0, 2)); + V13 = _mm_shuffle_ps(V13, D1, _MM_SHUFFLE(2, 1, 2, 1)); + + __m128 C0 = _mm_mul_ps(V00, V10); + __m128 C2 = _mm_mul_ps(V01, V11); + __m128 C4 = _mm_mul_ps(V02, V12); + __m128 C6 = _mm_mul_ps(V03, V13); + + // V11 = D0X,D0Y,D2X,D2X + V11 = _mm_shuffle_ps(D0, D2, _MM_SHUFFLE(0, 0, 1, 0)); + V00 = _mm_shuffle_ps(MT.m_rows[1], MT.m_rows[1], _MM_SHUFFLE(2, 1, 3, 2)); + V10 = _mm_shuffle_ps(D0, V11, _MM_SHUFFLE(2, 1, 0, 3)); + V01 = _mm_shuffle_ps(MT.m_rows[0], MT.m_rows[0], _MM_SHUFFLE(1, 3, 2, 3)); + V11 = _mm_shuffle_ps(D0, V11, _MM_SHUFFLE(0, 2, 1, 2)); + // V13 = D1X,D1Y,D2Z,D2Z + V13 = _mm_shuffle_ps(D1, D2, _MM_SHUFFLE(2, 2, 1, 0)); + V02 = _mm_shuffle_ps(MT.m_rows[3], MT.m_rows[3], _MM_SHUFFLE(2, 1, 3, 2)); + V12 = _mm_shuffle_ps(D1, V13, _MM_SHUFFLE(2, 1, 0, 3)); + V03 = _mm_shuffle_ps(MT.m_rows[2], MT.m_rows[2], _MM_SHUFFLE(1, 3, 2, 3)); + V13 = _mm_shuffle_ps(D1, V13, _MM_SHUFFLE(0, 2, 1, 2)); + + V00 = _mm_mul_ps(V00, V10); + V01 = _mm_mul_ps(V01, V11); + V02 = _mm_mul_ps(V02, V12); + V03 = _mm_mul_ps(V03, V13); + C0 = _mm_sub_ps(C0, V00); + C2 = _mm_sub_ps(C2, V01); + C4 = _mm_sub_ps(C4, V02); + C6 = _mm_sub_ps(C6, V03); + + V00 = _mm_shuffle_ps(MT.m_rows[1], MT.m_rows[1], _MM_SHUFFLE(0, 3, 0, 3)); + // V10 = D0Z,D0Z,D2X,D2Y + V10 = _mm_shuffle_ps(D0, D2, _MM_SHUFFLE(1, 0, 2, 2)); + V10 = _mm_shuffle_ps(V10, V10, _MM_SHUFFLE(0, 2, 3, 0)); + V01 = _mm_shuffle_ps(MT.m_rows[0], MT.m_rows[0], _MM_SHUFFLE(2, 0, 3, 1)); + // V11 = D0X,D0W,D2X,D2Y + V11 = _mm_shuffle_ps(D0, D2, _MM_SHUFFLE(1, 0, 3, 0)); + V11 = _mm_shuffle_ps(V11, V11, _MM_SHUFFLE(2, 1, 0, 3)); + V02 = _mm_shuffle_ps(MT.m_rows[3], MT.m_rows[3], _MM_SHUFFLE(0, 3, 0, 3)); + // V12 = D1Z,D1Z,D2Z,D2W + V12 = _mm_shuffle_ps(D1, D2, _MM_SHUFFLE(3, 2, 2, 2)); + V12 = _mm_shuffle_ps(V12, V12, _MM_SHUFFLE(0, 2, 3, 0)); + V03 = _mm_shuffle_ps(MT.m_rows[2], MT.m_rows[2], _MM_SHUFFLE(2, 0, 3, 1)); + // V13 = D1X,D1W,D2Z,D2W + V13 = _mm_shuffle_ps(D1, D2, _MM_SHUFFLE(3, 2, 3, 0)); + V13 = _mm_shuffle_ps(V13, V13, _MM_SHUFFLE(2, 1, 0, 3)); + + V00 = _mm_mul_ps(V00, V10); + V01 = _mm_mul_ps(V01, V11); + V02 = _mm_mul_ps(V02, V12); + V03 = _mm_mul_ps(V03, V13); + __m128 C1 = _mm_sub_ps(C0, V00); + C0 = _mm_add_ps(C0, V00); + __m128 C3 = _mm_add_ps(C2, V01); + C2 = _mm_sub_ps(C2, V01); + __m128 C5 = _mm_sub_ps(C4, V02); + C4 = _mm_add_ps(C4, V02); + __m128 C7 = _mm_add_ps(C6, V03); + C6 = _mm_sub_ps(C6, V03); + + C0 = _mm_shuffle_ps(C0, C1, _MM_SHUFFLE(3, 1, 2, 0)); + C2 = _mm_shuffle_ps(C2, C3, _MM_SHUFFLE(3, 1, 2, 0)); + C4 = _mm_shuffle_ps(C4, C5, _MM_SHUFFLE(3, 1, 2, 0)); + C6 = _mm_shuffle_ps(C6, C7, _MM_SHUFFLE(3, 1, 2, 0)); + C0 = _mm_shuffle_ps(C0, C0, _MM_SHUFFLE(3, 1, 2, 0)); + C2 = _mm_shuffle_ps(C2, C2, _MM_SHUFFLE(3, 1, 2, 0)); + C4 = _mm_shuffle_ps(C4, C4, _MM_SHUFFLE(3, 1, 2, 0)); + C6 = _mm_shuffle_ps(C6, C6, _MM_SHUFFLE(3, 1, 2, 0)); + + __m128 vTemp = Vector::Dot4(C0, MT.m_rows[0]); + vTemp = _mm_div_ps(Vector::One, vTemp); + m_rows[0] = _mm_mul_ps(C0, vTemp); + m_rows[1] = _mm_mul_ps(C2, vTemp); + m_rows[2] = _mm_mul_ps(C4, vTemp); + m_rows[3] = _mm_mul_ps(C6, vTemp); + return *this; + } + + inline Matrix Matrix::GetInverse() const + { + Matrix m = *this; + m.Invert(); + return m; + } + + inline Vector Matrix::GetDeterminant() const + { + Vector V0 = m_rows[2].Shuffle(1, 0, 0, 0); + Vector V1 = m_rows[3].Shuffle(2, 2, 1, 1); + Vector V2 = m_rows[2].Shuffle(1, 0, 0, 0); + Vector V3 = m_rows[3].Shuffle(3, 3, 3, 2); + Vector V4 = m_rows[2].Shuffle(2, 2, 1, 1); + Vector V5 = m_rows[3].Shuffle(3, 3, 3, 2); + + Vector P0 = V0 * V1; + Vector P1 = V2 * V3; + Vector P2 = V4 * V5; + + V0 = m_rows[2].Shuffle(2, 2, 1, 1); + V1 = m_rows[3].Shuffle(1, 0, 0, 0); + V2 = m_rows[2].Shuffle(3, 3, 3, 2); + V3 = m_rows[3].Shuffle(1, 0, 0, 0); + V4 = m_rows[2].Shuffle(3, 3, 3, 2); + V5 = m_rows[3].Shuffle(2, 2, 1, 1); + + P0 = Vector::NegativeMultiplySubtract(V0, V1, P0); + P1 = Vector::NegativeMultiplySubtract(V2, V3, P1); + P2 = Vector::NegativeMultiplySubtract(V4, V5, P2); + + V0 = m_rows[1].Shuffle(3, 3, 3, 2); + V1 = m_rows[1].Shuffle(2, 2, 1, 1); + V2 = m_rows[1].Shuffle(1, 0, 0, 0); + + static Vector const Sign(1.0f, -1.0f, 1.0f, -1.0f); + Vector S = m_rows[0] * Sign; + Vector R = V0 * P0; + R = Vector::NegativeMultiplySubtract(V1, P1, R); + R = Vector::MultiplyAdd(V2, P2, R); + + return Vector::Dot4(S, R); + } + + inline float Matrix::GetDeterminantAsFloat() const + { + return GetDeterminant().GetX(); + } + + inline Vector Matrix::GetTranslation() const + { + return m_rows[3].GetWithW0(); + } + + inline const Vector& Matrix::GetTranslationWithW() const + { + return m_rows[3]; + } + + inline Matrix& Matrix::SetTranslation(const Vector& v) + { + m_rows[3] = v.GetWithW1(); + return *this; + } + + inline Matrix& Matrix::SetTranslation(const Float3& v) + { + m_rows[3] = Vector(v, 1.0f); + return *this; + } + + inline Matrix& Matrix::SetTranslation(const Float4& v) + { + m_rows[3] = Vector(v.m_x, v.m_y, v.m_z, 1.0f); + return *this; + } + + inline Quaternion Matrix::GetRotation() const + { + // based on RTM: https://github.com/nfrechette/rtm + + const Vector& axisX = m_rows[0]; + const Vector& axisY = m_rows[1]; + const Vector& axisZ = m_rows[2]; + + // Zero scale is not supported + if (axisX.IsNearZero4() || axisY.IsNearZero4() || axisZ.IsNearZero4()) + { + HALT(); + } + + float const axisX_X = axisX.GetX(); + float const axisY_Y = axisY.GetY(); + float const axisZ_Z = axisZ.GetZ(); + + float const mtx_trace = axisX_X + axisY_Y + axisZ_Z; + if (mtx_trace > 0.0) + { + float const axisX_y = axisX.GetY(); + float const axisX_z = axisX.GetZ(); + + float const axisY_x = axisY.GetX(); + float const axisY_z = axisY.GetZ(); + + float const axisZ_x = axisZ.GetX(); + float const axisZ_y = axisZ.GetY(); + + float const inv_trace = Math::Reciprocal(Math::Sqrt(mtx_trace + 1.0f)); + float const half_inv_trace = inv_trace * 0.5f; + + float const m_x = (axisY_z - axisZ_y) * half_inv_trace; + float const m_y = (axisZ_x - axisX_z) * half_inv_trace; + float const m_z = (axisX_y - axisY_x) * half_inv_trace; + float const m_w = Math::Reciprocal(inv_trace) * 0.5f; + + return Quaternion(m_x, m_y, m_z, m_w).GetNormalized(); + } + else + { + // Find the axis with the highest diagonal value + int32_t axisIdx0 = 0; + if (axisY_Y > axisX_X) + { + axisIdx0 = 1; + } + + if (axisZ_Z > m_rows[axisIdx0][axisIdx0]) + { + axisIdx0 = 2; + } + + int32_t const axisIdx1 = (axisIdx0 + 1) % 3; + int32_t const axisIdx2 = (axisIdx1 + 1) % 3; + + float const pseudoTrace = 1.0f + m_rows[axisIdx0][axisIdx0] - m_rows[axisIdx1][axisIdx1] - m_rows[axisIdx2][axisIdx2]; + float const inversePseudoTrace = Math::Reciprocal(Math::Sqrt(pseudoTrace)); + float const halfInversePseudoTrace = inversePseudoTrace * 0.5f; + + Float4 rawQuatValues; + rawQuatValues[axisIdx0] = Math::Reciprocal(inversePseudoTrace) * 0.5f; + rawQuatValues[axisIdx1] = halfInversePseudoTrace * (m_rows[axisIdx0][axisIdx1] + m_rows[axisIdx1][axisIdx0]); + rawQuatValues[axisIdx2] = halfInversePseudoTrace * (m_rows[axisIdx0][axisIdx2] + m_rows[axisIdx2][axisIdx0]); + rawQuatValues[3] = halfInversePseudoTrace * (m_rows[axisIdx1][axisIdx2] - m_rows[axisIdx2][axisIdx1]); + return Quaternion(rawQuatValues).GetNormalized(); + } + } + + inline Matrix& Matrix::SetRotation(const Matrix& rotation) + { + ASSERT(Math::Abs(rotation.GetDeterminant().GetX()) == 1.0f); + m_rows[0] = rotation.m_rows[0]; + m_rows[1] = rotation.m_rows[1]; + m_rows[2] = rotation.m_rows[2]; + return *this; + } + + inline Matrix& Matrix::SetRotation(const Quaternion& rotation) + { + static __m128 const constant1110 = { 1.0f, 1.0f, 1.0f, 0.0f }; + + __m128 Q0 = _mm_add_ps(rotation, rotation); + __m128 Q1 = _mm_mul_ps(rotation, Q0); + + __m128 V0 = _mm_shuffle_ps(Q1, Q1, _MM_SHUFFLE(3, 0, 0, 1)); + V0 = _mm_and_ps(V0, SIMD::g_maskXYZ0); + __m128 V1 = _mm_shuffle_ps(Q1, Q1, _MM_SHUFFLE(3, 1, 2, 2)); + V1 = _mm_and_ps(V1, SIMD::g_maskXYZ0); + __m128 R0 = _mm_sub_ps(constant1110, V0); + R0 = _mm_sub_ps(R0, V1); + + V0 = _mm_shuffle_ps(rotation, rotation, _MM_SHUFFLE(3, 1, 0, 0)); + V1 = _mm_shuffle_ps(Q0, Q0, _MM_SHUFFLE(3, 2, 1, 2)); + V0 = _mm_mul_ps(V0, V1); + + V1 = _mm_shuffle_ps(rotation, rotation, _MM_SHUFFLE(3, 3, 3, 3)); + __m128 V2 = _mm_shuffle_ps(Q0, Q0, _MM_SHUFFLE(3, 0, 2, 1)); + V1 = _mm_mul_ps(V1, V2); + + __m128 R1 = _mm_add_ps(V0, V1); + __m128 R2 = _mm_sub_ps(V0, V1); + + V0 = _mm_shuffle_ps(R1, R2, _MM_SHUFFLE(1, 0, 2, 1)); + V0 = _mm_shuffle_ps(V0, V0, _MM_SHUFFLE(1, 3, 2, 0)); + V1 = _mm_shuffle_ps(R1, R2, _MM_SHUFFLE(2, 2, 0, 0)); + V1 = _mm_shuffle_ps(V1, V1, _MM_SHUFFLE(2, 0, 2, 0)); + + Q1 = _mm_shuffle_ps(R0, V0, _MM_SHUFFLE(1, 0, 3, 0)); + Q1 = _mm_shuffle_ps(Q1, Q1, _MM_SHUFFLE(1, 3, 2, 0)); + + m_rows[0] = Q1; + + Q1 = _mm_shuffle_ps(R0, V0, _MM_SHUFFLE(3, 2, 3, 1)); + Q1 = _mm_shuffle_ps(Q1, Q1, _MM_SHUFFLE(1, 3, 0, 2)); + m_rows[1] = Q1; + + Q1 = _mm_shuffle_ps(V1, R0, _MM_SHUFFLE(3, 2, 1, 0)); + m_rows[2] = Q1; + return *this; + } + + inline Matrix& Matrix::SetRotationMaintainingScale(const Matrix& rotation) + { + Vector const scale = GetScale(); + SetRotation(rotation); + return SetScale(scale); + } + + inline Matrix& Matrix::SetRotationMaintainingScale(const Quaternion& rotation) + { + Vector const scale = GetScale(); + SetRotation(rotation); + return SetScale(scale); + } + + inline Matrix& Matrix::SetScale(float uniformScale) + { + SetScale(Vector(uniformScale)); + return *this; + } + + inline Matrix& Matrix::RemoveScaleFast() + { + m_rows[0] = m_rows[0].GetNormalized4(); + m_rows[1] = m_rows[1].GetNormalized4(); + m_rows[2] = m_rows[2].GetNormalized4(); + return *this; + } + + inline Matrix& Matrix::SetScaleFast(const Vector& scale) + { + m_rows[0] = m_rows[0].GetNormalized3() * scale.GetSplatX(); + m_rows[1] = m_rows[1].GetNormalized3() * scale.GetSplatY(); + m_rows[2] = m_rows[2].GetNormalized3() * scale.GetSplatZ(); + return *this; + } + + inline Matrix& Matrix::SetScaleFast(float uniformScale) + { + SetScaleFast(Vector(uniformScale)); + return *this; + } + + inline Vector Matrix::RotateVector(const Vector& vector) const + { + Vector const X = vector.GetSplatX(); + Vector const Y = vector.GetSplatY(); + Vector const Z = vector.GetSplatZ(); + + Vector Result = Z * m_rows[2]; + Result = Vector::MultiplyAdd(Y, m_rows[1], Result); + Result = Vector::MultiplyAdd(X, m_rows[0], Result); + + return Result; + } + + inline Vector Matrix::TransformNormal(const Vector& vector) const + { + return RotateVector(vector); + } + + inline Vector Matrix::TransformPoint(const Vector& point) const + { + Vector const X = point.GetSplatX(); + Vector const Y = point.GetSplatY(); + Vector const Z = point.GetSplatZ(); + + Vector result = Vector::MultiplyAdd(Z, m_rows[2], m_rows[3]); + result = Vector::MultiplyAdd(Y, m_rows[1], result); + result = Vector::MultiplyAdd(X, m_rows[0], result); + + Vector const W = result.GetSplatW(); + return result / W; + } + + inline Vector Matrix::TransformVector3(const Vector& V) const + { + Vector const X = V.GetSplatX(); + Vector const Y = V.GetSplatY(); + Vector const Z = V.GetSplatZ(); + + Vector result = Vector::MultiplyAdd(Z, m_rows[2], m_rows[3]); + result = Vector::MultiplyAdd(Y, m_rows[1], result); + result = Vector::MultiplyAdd(X, m_rows[0], result); + + return result; + } + + inline Vector Matrix::TransformVector4(const Vector& V) const + { + // Splat m_x,m_y,m_z and m_w + Vector vTempX = V.GetSplatX(); + Vector vTempY = V.GetSplatY(); + Vector vTempZ = V.GetSplatZ(); + Vector vTempW = V.GetSplatW(); + + // Mul by the matrix + vTempX = _mm_mul_ps(vTempX, m_rows[0]); + vTempY = _mm_mul_ps(vTempY, m_rows[1]); + vTempZ = _mm_mul_ps(vTempZ, m_rows[2]); + vTempW = _mm_mul_ps(vTempW, m_rows[3]); + + // Add them all together + vTempX = _mm_add_ps(vTempX, vTempY); + vTempZ = _mm_add_ps(vTempZ, vTempW); + vTempX = _mm_add_ps(vTempX, vTempZ); + + return vTempX; + } + + inline Vector& Matrix::operator[](uint32_t i) + { + ASSERT(i < 4); + return m_rows[i]; + } + + inline const Vector Matrix::operator[](uint32_t i) const + { + ASSERT(i < 4); + return m_rows[i]; + } + + inline Matrix Matrix::operator*(const Matrix& rhs) const + { + Matrix result = *this; + result *= rhs; + return result; + } + + inline Matrix& Matrix::operator*= (const Matrix& rhs) + { + Vector vX, vY, vZ, vW; + + // Use vW to hold the original row + vW = m_rows[0]; + vX = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(0, 0, 0, 0)); + vY = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(1, 1, 1, 1)); + vZ = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(2, 2, 2, 2)); + vW = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(3, 3, 3, 3)); + vX = _mm_mul_ps(vX, rhs.m_rows[0]); + vY = _mm_mul_ps(vY, rhs.m_rows[1]); + vZ = _mm_mul_ps(vZ, rhs.m_rows[2]); + vW = _mm_mul_ps(vW, rhs.m_rows[3]); + vX = _mm_add_ps(vX, vZ); + vY = _mm_add_ps(vY, vW); + vX = _mm_add_ps(vX, vY); + m_rows[0] = vX; + + // Repeat for the other 3 rows + vW = m_rows[1]; + vX = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(0, 0, 0, 0)); + vY = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(1, 1, 1, 1)); + vZ = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(2, 2, 2, 2)); + vW = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(3, 3, 3, 3)); + vX = _mm_mul_ps(vX, rhs.m_rows[0]); + vY = _mm_mul_ps(vY, rhs.m_rows[1]); + vZ = _mm_mul_ps(vZ, rhs.m_rows[2]); + vW = _mm_mul_ps(vW, rhs.m_rows[3]); + vX = _mm_add_ps(vX, vZ); + vY = _mm_add_ps(vY, vW); + vX = _mm_add_ps(vX, vY); + m_rows[1] = vX; + + vW = m_rows[2]; + vX = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(0, 0, 0, 0)); + vY = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(1, 1, 1, 1)); + vZ = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(2, 2, 2, 2)); + vW = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(3, 3, 3, 3)); + vX = _mm_mul_ps(vX, rhs.m_rows[0]); + vY = _mm_mul_ps(vY, rhs.m_rows[1]); + vZ = _mm_mul_ps(vZ, rhs.m_rows[2]); + vW = _mm_mul_ps(vW, rhs.m_rows[3]); + vX = _mm_add_ps(vX, vZ); + vY = _mm_add_ps(vY, vW); + vX = _mm_add_ps(vX, vY); + m_rows[2] = vX; + + vW = m_rows[3]; + vX = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(0, 0, 0, 0)); + vY = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(1, 1, 1, 1)); + vZ = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(2, 2, 2, 2)); + vW = _mm_shuffle_ps(vW, vW, _MM_SHUFFLE(3, 3, 3, 3)); + vX = _mm_mul_ps(vX, rhs.m_rows[0]); + vY = _mm_mul_ps(vY, rhs.m_rows[1]); + vZ = _mm_mul_ps(vZ, rhs.m_rows[2]); + vW = _mm_mul_ps(vW, rhs.m_rows[3]); + vX = _mm_add_ps(vX, vZ); + vY = _mm_add_ps(vY, vW); + vX = _mm_add_ps(vX, vY); + m_rows[3] = vX; + return *this; + } + + inline Matrix Matrix::operator*(const Quaternion& rhs) const + { + return operator*(Matrix(rhs)); + } + + inline Matrix Matrix::operator*=(const Quaternion& rhs) + { + return operator*=(Matrix(rhs)); + } + + inline bool Matrix::operator==(const Matrix& rhs) const + { + for (auto i = 0; i < 4; i++) + { + for (auto j = 0; j < 4; j++) + { + if (m_values[i][j] != rhs.m_values[i][j]) + { + return false; + } + } + } + + return true; + } +} diff --git a/MotionCorrection/src/cpp/Math/Quaternion.cpp b/MotionCorrection/src/cpp/Math/Quaternion.cpp new file mode 100644 index 0000000000000000000000000000000000000000..230dfc7e466714949caab4e8472a5d820cd2ae56 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Quaternion.cpp @@ -0,0 +1,24 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "Quaternion.h" +#include "Matrix.h" + +namespace Math +{ + Quaternion const Quaternion::Identity(0, 0, 0, 1); + + // Rotation order is XYZ + EulerAngles Quaternion::ToEulerAngles() const + { + return Matrix(*this).ToEulerAngles(); + } + + Quaternion Quaternion::LookRotation(const Vector& forward, const Vector& up) + { + const Vector t = Vector::Cross3(up, forward).Normalize3(); + return Matrix(t, Vector::Cross3(forward, t), forward).GetRotation(); + } +} diff --git a/MotionCorrection/src/cpp/Math/Quaternion.h b/MotionCorrection/src/cpp/Math/Quaternion.h new file mode 100644 index 0000000000000000000000000000000000000000..9ca72f3d8bdf7be30bda44b6302160a5377df287 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Quaternion.h @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Vector.h" + +namespace Math +{ + class alignas(16) Quaternion + { + public: + + static Quaternion const Identity; + + // Calculate the rotation required to align the source vector to the target vector (shortest path) + static Quaternion FromRotationBetweenNormalizedVectors(const Vector& sourceVector, const Vector& targetVector); + + // Calculate the rotation required to align one vector onto another but also taking account a fallback rotation axis for opposite parallel vectors + static Quaternion FromRotationBetweenNormalizedVectors(const Vector& sourceVector, const Vector& targetVector, const Vector& fallbackRotationAxis); + + // Calculate the rotation required to align the source vector to the target vector (shortest path) + static Quaternion FromRotationBetweenVectors(const Vector& sourceVector, const Vector& targetVector); + + // Normalized LERP - not accurate - only use for really short distances + static Quaternion NLerp(const Quaternion& from, const Quaternion& to, float t); + + // Standard and accurate Spherical LERP - based on DirectX Math + static Quaternion SLerp(const Quaternion& from, const Quaternion& to, float t); + + // Fast approximation of a Spherical LERP - based on "A fast and accurate estimate for SLERP" by David Eberly + static Quaternion FastSLerp(const Quaternion& from, const Quaternion& to, float t); + + // Spherical quadrangle/cubic interpolation for quaternions + static Quaternion SQuad(const Quaternion& q0, const Quaternion& q1, const Quaternion& q2, const Quaternion& q3, float t); + + // Calculate the shortest delta quaternion needed to rotate 'from' onto 'to' + static Quaternion Delta(const Quaternion& from, const Quaternion& to); + + // Simple vector dot product between two quaternions + static Vector Dot(const Quaternion& q0, const Quaternion& q1); + + // Calculate the angular distance between two quaternions + static Radians Distance(const Quaternion& q0, const Quaternion& q1); + + // Calculate look rotation given forward and up vectors + static Quaternion LookRotation(const Vector& forward, const Vector& up); + + public: + + Quaternion() = default; + explicit Quaternion(NoInit_t); + explicit Quaternion(IdentityInit_t); + explicit Quaternion(const Vector& v); + explicit Quaternion(float ix, float iy, float iz, float iw); + explicit Quaternion(const Float4& v); + + explicit Quaternion(const Vector& axis, Radians angle); + explicit Quaternion(AxisAngle axisAngle); + + explicit Quaternion(const EulerAngles& eulerAngles); + explicit Quaternion(Radians rotX, Radians rotY, Radians rotZ); + + operator __m128& (); + operator const __m128& () const; + + Float4 ToFloat4() const; + Vector ToVector() const; + + Vector Length(); + float GetLength() const; + + // Get the angle this rotation represents around the specified axis + Radians GetAngle() const; + + AxisAngle ToAxisAngle() const; + EulerAngles ToEulerAngles() const; + + Vector RotateVector(const Vector& vector) const; + Vector RotateVectorInverse(const Vector& vector) const; + + Quaternion& Conjugate(); + Quaternion GetConjugate() const; + + Quaternion& Negate(); + Quaternion GetNegated() const; + + Quaternion& Invert(); + Quaternion GetInverse() const; + + Quaternion& Normalize(); + Quaternion GetNormalized() const; + + Vector XAxis() const noexcept; + Vector YAxis() const noexcept; + Vector ZAxis() const noexcept; + + // Ensure that this rotation is the shortest in terms of the angle (i.e. -5 instead of 355) + Quaternion& MakeShortestPath(); + + // Ensure that this rotation is the shortest in terms of the angle (i.e. -5 instead of 355) + Quaternion GetShortestPath() const; + + // This function will return the estimated normalized quaternion, this is not super accurate but a lot faster (use with care) + Quaternion& NormalizeInaccurate(); + + // This function will return the estimated normalized quaternion, this is not super accurate but a lot faster (use with care) + Quaternion GetNormalizedInaccurate() const; + + bool IsNormalized() const; + bool IsIdentity() const; + + // Concatenate the rotation of this onto rhs and return the result i.e. first rotate by rhs then by this + // This means order of rotation is right-to-left: child-rotation * parent-rotation + Quaternion operator*(const Quaternion& rhs) const; + Quaternion& operator*=(const Quaternion& rhs); + + // Is the distance between this quaternion and another one under the threshold? + bool IsNearEqual(const Quaternion& rhs, Radians const threshold = Math::DegreesToRadians) const; + + // Exact equality + bool operator==(const Quaternion& rhs) const; + + // Exact equality + bool operator!=(const Quaternion& rhs) const; + + private: + + Vector GetSplatW() const; + float GetW() const; + + Quaternion& operator=(const Vector& v) = delete; + + public: + + __m128 m_data; + }; + + static_assert(sizeof(Vector) == 16, "Quaternion size must be 16 bytes!"); +} + +#include "Quaternion.inl" diff --git a/MotionCorrection/src/cpp/Math/Quaternion.inl b/MotionCorrection/src/cpp/Math/Quaternion.inl new file mode 100644 index 0000000000000000000000000000000000000000..fe5aa9e692d5fa4c1f286b4b9058e557b4273cb8 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Quaternion.inl @@ -0,0 +1,606 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Quaternion.h" + +namespace Math +{ + inline Quaternion Quaternion::FromRotationBetweenNormalizedVectors(const Vector& from, const Vector& to) + { + ASSERT(from.IsNormalized3() && to.IsNormalized3()); + + Quaternion result; + + // Parallel vectors - return zero rotation + Vector const dot = Vector::Dot3(from, to); + if (dot.IsGreaterThanEqual4(Vector::OneMinusEpsilon)) + { + result = Quaternion::Identity; + } + // Opposite vectors - return 180 rotation around any orthogonal axis + else if (dot.IsLessThanEqual4(Vector::EpsilonMinusOne)) + { + Float4 const fromValues = from.ToFloat4(); + result = Quaternion(-fromValues.m_z, fromValues.m_y, fromValues.m_x, 0); + result.Normalize(); + } + else // Calculate quaternion rotation + { + Vector const cross = Vector::Cross3(from, to); + Vector Q = Vector::Select(cross, dot, Vector::Select0001); + Q += Vector::Select(Vector::Zero, Q.Length4(), Vector::Select0001); + result = Quaternion(Q); + result.Normalize(); + } + + return result; + } + + inline Quaternion Quaternion::FromRotationBetweenNormalizedVectors(const Vector& from, const Vector& to, const Vector& fallbackRotationAxis) + { + ASSERT(from.IsNormalized3() && to.IsNormalized3()); + + Quaternion Q(NoInit); + + Vector rotationAxis = from.Cross3(to).GetNormalized3(); + if (rotationAxis.GetLengthSquared3() == 0) + { + rotationAxis = fallbackRotationAxis; + } + + float const dot = from.GetDot3(to); + if (dot >= (1.0f - Math::Epsilon)) + { + Q = Quaternion::Identity; + } + else + { + float const angle = Math::ACos(dot); + Q = Quaternion(rotationAxis, angle); + } + + return Q; + } + + inline Quaternion Quaternion::FromRotationBetweenVectors(const Vector& sourceVector, const Vector& targetVector) + { + return FromRotationBetweenNormalizedVectors( + sourceVector.GetNormalized3(), + targetVector.GetNormalized3()); + } + + inline Quaternion Quaternion::NLerp(const Quaternion& from, const Quaternion& to, float T) + { + ASSERT(T >= 0.0f && T <= 1.0f); + + Quaternion adjustedFrom(from); + + // Ensure that the rotations are in the same direction + if (Quaternion::Dot(from, to).IsLessThan4(Vector::Zero)) + { + adjustedFrom.Negate(); + } + + Quaternion result(Vector::Lerp(adjustedFrom.ToVector(), to.ToVector(), T)); + result.Normalize(); + return result; + } + + inline Quaternion Quaternion::SLerp(const Quaternion& from, const Quaternion& to, float T) + { + ASSERT(T >= 0.0f && T <= 1.0f); + + static SIMD::UIntMask const maskSign = { 0x80000000,0x00000000,0x00000000,0x00000000 }; + static __m128 const oneMinusEpsilon = { 1.0f - 0.00001f, 1.0f - 0.00001f, 1.0f - 0.00001f, 1.0f - 0.00001f }; + + Vector const VecT(T); + + Vector cosOmega = Quaternion::Dot(from, to); + + Vector control = cosOmega.LessThan(Vector::Zero); + Vector sign = Vector::Select(Vector::One, Vector::NegativeOne, control); + + cosOmega = _mm_mul_ps(cosOmega, sign); + control = cosOmega.LessThan(oneMinusEpsilon); + + Vector sinOmega = _mm_mul_ps(cosOmega, cosOmega); + sinOmega = _mm_sub_ps(Vector::One, sinOmega); + sinOmega = _mm_sqrt_ps(sinOmega); + + Vector omega = Vector::ATan2(sinOmega, cosOmega); + + Vector V01 = _mm_shuffle_ps(VecT, VecT, _MM_SHUFFLE(2, 3, 0, 1)); + V01 = _mm_and_ps(V01, SIMD::g_maskXY00); + V01 = _mm_xor_ps(V01, maskSign); + V01 = _mm_add_ps(Vector::UnitX, V01); + + Vector S0 = _mm_mul_ps(V01, omega); + S0 = Vector::Sin(S0); + S0 = _mm_div_ps(S0, sinOmega); + S0 = Vector::Select(V01, S0, control); + + Vector S1 = S0.GetSplatY(); + S0 = S0.GetSplatX(); + + S1 = _mm_mul_ps(S1, sign); + Vector result = _mm_mul_ps(from, S0); + S1 = _mm_mul_ps(S1, to); + result = _mm_add_ps(result, S1); + + return Quaternion(result); + } + + inline Quaternion Quaternion::FastSLerp(const Quaternion& q0, const Quaternion& q1, float t) + { + // Precomputed constants + constexpr float const mu = 1.85298109240830f; + static Vector const u0123 = _mm_setr_ps(1.f / (1 * 3), 1.f / (2 * 5), 1.f / (3 * 7), 1.f / (4 * 9)); + static Vector const u4567 = _mm_setr_ps(1.f / (5 * 11), 1.f / (6 * 13), 1.f / (7 * 15), mu / (8 * 17)); + static Vector const v0123 = _mm_setr_ps(1.f / 3, 2.f / 5, 3.f / 7, 4.f / 9); + static Vector const v4567 = _mm_setr_ps(5.f / 11, 6.f / 13, 7.f / 15, mu * 8 / 17); + static Vector const vSignMask = _mm_set1_ps(-0.f); + + // Common code for computing the scalar coefficients of SLERP + auto CalculateCoefficient = [](Vector vT, Vector xm1) + { + Vector const vTSquared = vT * vT; + + // ( b4, b5, b6, b7 ) = ( x-1 ) * ( u4 * t^2 - v4, u5 * t^2 - v5, u6 * t^2 - v6, u7 * t^2 - v7 ) + Vector b4567 = Vector::MultiplySubtract(u4567, vTSquared, v4567); + b4567 *= xm1; + + // ( b7, b7, b7, b7 ) + Vector b = b4567.GetSplatW(); + Vector c = b + Vector::One; + + // ( b6, b6, b6, b6 ) + b = b4567.GetSplatZ(); + c = Vector::MultiplyAdd(b, c, Vector::One); + + // ( b5, b5, b5, b5 ) + b = b4567.GetSplatY(); + c = Vector::MultiplyAdd(b, c, Vector::One); + + // ( b4, b4, b4, b4 ) + b = b4567.GetSplatX(); + c = Vector::MultiplyAdd(b, c, Vector::One); + + // ( b0, b1, b2, b3 ) = + // ( x-1)*(u0* t^2-v0, u1 * t^2 -v1, u2* t^2-v2, u3* t^2-v3 ) + Vector b0123 = Vector::MultiplySubtract(u0123, vTSquared, v0123); + b0123 *= xm1; + + // ( b3, b3, b3, b3 ) + b = b0123.GetSplatW(); + c = Vector::MultiplyAdd(b, c, Vector::One); + + // ( b2, b2, b2, b2 ) + b = b0123.GetSplatZ(); + c = Vector::MultiplyAdd(b, c, Vector::One); + + // ( b1, b1, b1, b1 ) + b = b0123.GetSplatY(); + c = Vector::MultiplyAdd(b, c, Vector::One); + + // ( b0, b0, b0, b0 ) + b = b0123.GetSplatX(); + c = Vector::MultiplyAdd(b, c, Vector::One); + c *= vT; + + return c; + }; + + Vector x = Vector::Dot4(q0.m_data, q1.m_data); // cos ( theta ) in all components + + Vector sign = _mm_and_ps(vSignMask, x); + x = _mm_xor_ps(sign, x); + Vector localQ1 = _mm_xor_ps(sign, q1); + + Vector xm1 = x - Vector::One; + + Vector cT = CalculateCoefficient(Vector(t), xm1); + Vector cD = CalculateCoefficient(Vector(1.0f - t), xm1); + cT = cT * localQ1; + + Quaternion result(Vector::MultiplyAdd(cD, q0.m_data, cT)); + return result; + } + + inline Quaternion Quaternion::SQuad(const Quaternion& q0, const Quaternion& q1, const Quaternion& q2, const Quaternion& q3, float t) + { + ASSERT(t >= 0.0f && t <= 1.0f); + + Quaternion const q03 = Quaternion::SLerp(q0, q3, t); + Quaternion const q12 = Quaternion::SLerp(q1, q2, t); + t = (t - (t * t)) * 2; + Quaternion const result = Quaternion::SLerp(q03, q12, t); + return result; + } + + inline Quaternion Quaternion::Delta(const Quaternion& from, const Quaternion& to) + { + return to * from.GetInverse(); + } + + inline Vector Quaternion::Dot(const Quaternion& q0, const Quaternion& q1) + { + return Vector::Dot4(q0.m_data, q1.m_data); + } + + inline Radians Quaternion::Distance(const Quaternion& q0, const Quaternion& q1) + { + float const dot = Math::Clamp(Dot(q0, q1).ToFloat(), -1.0f, 1.0f); + return Radians(2 * Math::ACos(Math::Abs(dot))); + } + + inline Quaternion::Quaternion(NoInit_t) + { + } + + inline Quaternion::Quaternion(IdentityInit_t) + : m_data(Vector::UnitW.m_data) + { + } + + inline Quaternion::Quaternion(const Vector& v) + : m_data(v.m_data) + { + } + + inline Quaternion::Quaternion(float ix, float iy, float iz, float iw) + { + m_data = _mm_set_ps(iw, iz, iy, ix); + } + + inline Quaternion::Quaternion(const Float4& v) + : Quaternion(v.m_x, v.m_y, v.m_z, v.m_w) + { + } + + inline Quaternion::Quaternion(const Vector& axis, Radians angle) + { + ASSERT(axis.IsNormalized3()); + + auto N = _mm_and_ps(axis, SIMD::g_maskXYZ0); + N = _mm_or_ps(N, Vector::UnitW); + auto scale = _mm_set_ps1(0.5f * (float)angle); + + Vector sine, cosine; + Vector::SinCos(sine, cosine, scale); + + scale = _mm_and_ps(sine, SIMD::g_maskXYZ0); + cosine = _mm_and_ps(cosine, SIMD::g_mask000W); + scale = _mm_or_ps(scale, cosine); + + N = _mm_mul_ps(N, scale); + m_data = N; + } + + inline Quaternion::Quaternion(AxisAngle axisAngle) + : Quaternion(Vector(axisAngle.m_axis), axisAngle.m_angle) + { + } + + inline Quaternion::Quaternion(const EulerAngles& eulerAngles) + { + auto const rotationX = Quaternion(Vector::UnitX, eulerAngles.m_x); + auto const rotationY = Quaternion(Vector::UnitY, eulerAngles.m_y); + auto const rotationZ = Quaternion(Vector::UnitZ, eulerAngles.m_z); + + // Rotation order is XYZ - all in global space, hence the order is reversed + m_data = (rotationX * rotationY * rotationZ).GetNormalized().m_data; + } + + inline Quaternion::Quaternion(Radians rotX, Radians rotY, Radians rotZ) + : Quaternion(EulerAngles(rotX, rotY, rotZ)) + { + } + + inline Quaternion::operator __m128& () + { + return m_data; + } + + inline Quaternion::operator const __m128& () const + { + return m_data; + } + + inline Float4 Quaternion::ToFloat4() const + { + Float4 v; + _mm_storeu_ps(&v.m_x, m_data); + return v; + } + + inline Vector Quaternion::ToVector() const + { + return Vector(m_data); + } + + inline Vector Quaternion::Length() + { + return ToVector().Length4(); + } + + inline float Quaternion::GetLength() const + { + return ToVector().GetLength4(); + } + + inline Radians Quaternion::GetAngle() const + { + return Radians(2.0f * Math::ACos(GetW())); + } + + inline AxisAngle Quaternion::ToAxisAngle() const + { + return AxisAngle(ToVector(), Radians(2.0f * Math::ACos(GetW()))); + } + + inline Vector Quaternion::RotateVector(const Vector& vector) const + { + Quaternion const A(Vector::Select(Vector::Select1110, vector, Vector::Select1110)); + Quaternion const result = GetConjugate() * A; + return (result * *this).ToVector(); + } + + inline Vector Quaternion::RotateVectorInverse(const Vector& vector) const + { + Quaternion const A(Vector::Select(Vector::Select1110, vector, Vector::Select1110)); + Quaternion const result = *this * A; + return (result * GetConjugate()).ToVector(); + } + + inline Quaternion& Quaternion::Conjugate() + { + static __m128 const conj = { -1.0f, -1.0f, -1.0f, 1.0f }; + m_data = _mm_mul_ps(*this, conj); + return *this; + } + + inline Quaternion Quaternion::GetConjugate() const + { + Quaternion q = *this; + q.Conjugate(); + return q; + } + inline Quaternion& Quaternion::Negate() + { + m_data = _mm_mul_ps(*this, Vector::NegativeOne); + return *this; + } + + inline Quaternion Quaternion::GetNegated() const + { + Quaternion q = *this; + q.Negate(); + return q; + } + + inline Quaternion& Quaternion::Invert() + { + Vector const conjugate(GetConjugate().m_data); + Vector const length = ToVector().Length4(); + Vector const mask = length.LessThanEqual(Vector::Epsilon); + Vector const result = conjugate / length; + m_data = result.Select(result, Vector::Zero, mask); + return *this; + } + + inline Quaternion Quaternion::GetInverse() const + { + Quaternion q = *this; + q.Invert(); + return q; + } + + inline Quaternion& Quaternion::Normalize() + { + m_data = ToVector().GetNormalized4().m_data; + return *this; + } + + inline Quaternion Quaternion::GetNormalized() const + { + Quaternion q = *this; + q.Normalize(); + return q; + } + + inline Vector Quaternion::XAxis() const noexcept + { + const float x = _mm_cvtss_f32(m_data); + const float y = _mm_cvtss_f32( + _mm_shuffle_ps(m_data, m_data, + _MM_SHUFFLE(1, 1, 1, 1))); + const float z = _mm_cvtss_f32( + _mm_shuffle_ps(m_data, m_data, + _MM_SHUFFLE(2, 2, 2, 2))); + const float w = _mm_cvtss_f32( + _mm_shuffle_ps(m_data, m_data, + _MM_SHUFFLE(3, 3, 3, 3))); + + const float s = 2.0f * w; + const float x2 = 2.0f * x; + + return Vector( + x2 * x + s * w - 1.0f, + x2 * y + s * z, + x2 * z + s * -y); + } + + inline Vector Quaternion::YAxis() const noexcept + { + const float x = _mm_cvtss_f32(m_data); + const float y = _mm_cvtss_f32( + _mm_shuffle_ps(m_data, m_data, + _MM_SHUFFLE(1, 1, 1, 1))); + const float z = _mm_cvtss_f32( + _mm_shuffle_ps(m_data, m_data, + _MM_SHUFFLE(2, 2, 2, 2))); + const float w = _mm_cvtss_f32( + _mm_shuffle_ps(m_data, m_data, + _MM_SHUFFLE(3, 3, 3, 3))); + + const float s = 2.0f * w; + const float y2 = 2.0f * y; + + return Vector( + y2 * x + s * -z, + y2 * y + s * w - 1.0f, + y2 * z + s * x); + } + + inline Vector Quaternion::ZAxis() const noexcept + { + const float x = _mm_cvtss_f32(m_data); + const float y = _mm_cvtss_f32( + _mm_shuffle_ps(m_data, m_data, + _MM_SHUFFLE(1, 1, 1, 1))); + const float z = _mm_cvtss_f32( + _mm_shuffle_ps(m_data, m_data, + _MM_SHUFFLE(2, 2, 2, 2))); + const float w = _mm_cvtss_f32( + _mm_shuffle_ps(m_data, m_data, + _MM_SHUFFLE(3, 3, 3, 3))); + + const float s = 2.0f * w; + const float z2 = 2.0f * z; + + return Vector( + x * z2 + s * y, + y * z2 + s * -x, + z * z2 + s * w - 1.0f); + } + + inline Quaternion& Quaternion::MakeShortestPath() + { + // If we have a > 180 angle, negate + // w < 0.0f is the same as dot( identity, q ) < 0 + if (GetW() < 0.0f) + { + Negate(); + } + + return *this; + } + + inline Quaternion Quaternion::GetShortestPath() const + { + Quaternion sp = *this; + sp.MakeShortestPath(); + return sp; + } + + inline Quaternion& Quaternion::NormalizeInaccurate() + { + *this = GetNormalizedInaccurate(); + return *this; + } + + inline Quaternion Quaternion::GetNormalizedInaccurate() const + { + __m128 vLengthSq = _mm_mul_ps(m_data, m_data); + __m128 vTemp = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(3, 2, 3, 2)); + vLengthSq = _mm_add_ps(vLengthSq, vTemp); + vLengthSq = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(1, 0, 0, 0)); + vTemp = _mm_shuffle_ps(vTemp, vLengthSq, _MM_SHUFFLE(3, 3, 0, 0)); + vLengthSq = _mm_add_ps(vLengthSq, vTemp); + vLengthSq = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(2, 2, 2, 2)); + + // Get the reciprocal and mul to perform the normalization + Quaternion result; + result.m_data = _mm_rsqrt_ps(vLengthSq); + result.m_data = _mm_mul_ps(result.m_data, m_data); + return result; + } + + inline bool Quaternion::IsNormalized() const + { + return ToVector().IsNormalized4(); + } + + inline bool Quaternion::IsIdentity() const + { + return ToVector().IsEqual3(Vector::UnitW); + } + + inline Quaternion Quaternion::operator*(const Quaternion& rhs) const + { + static const __m128 controlWZYX = { 1.0f,-1.0f, 1.0f,-1.0f }; + static const __m128 controlZWXY = { 1.0f, 1.0f,-1.0f,-1.0f }; + static const __m128 controlYXWZ = { -1.0f, 1.0f, 1.0f,-1.0f }; + + // Copy to SSE registers and use as few as possible for x86 + __m128 Q2X = rhs; + __m128 Q2Y = rhs; + __m128 Q2Z = rhs; + __m128 vResult = rhs; + // Splat with one instruction + vResult = _mm_shuffle_ps(vResult, vResult, _MM_SHUFFLE(3, 3, 3, 3)); + Q2X = _mm_shuffle_ps(Q2X, Q2X, _MM_SHUFFLE(0, 0, 0, 0)); + Q2Y = _mm_shuffle_ps(Q2Y, Q2Y, _MM_SHUFFLE(1, 1, 1, 1)); + Q2Z = _mm_shuffle_ps(Q2Z, Q2Z, _MM_SHUFFLE(2, 2, 2, 2)); + // Retire Q1 and perform Q1*Q2W + vResult = _mm_mul_ps(vResult, *this); + __m128 Q1Shuffle = *this; + // Shuffle the copies of Q1 + Q1Shuffle = _mm_shuffle_ps(Q1Shuffle, Q1Shuffle, _MM_SHUFFLE(0, 1, 2, 3)); + // Mul by Q1WZYX + Q2X = _mm_mul_ps(Q2X, Q1Shuffle); + Q1Shuffle = _mm_shuffle_ps(Q1Shuffle, Q1Shuffle, _MM_SHUFFLE(2, 3, 0, 1)); + // Flip the signs on m_y and m_z + Q2X = _mm_mul_ps(Q2X, controlWZYX); + // Mul by Q1ZWXY + Q2Y = _mm_mul_ps(Q2Y, Q1Shuffle); + Q1Shuffle = _mm_shuffle_ps(Q1Shuffle, Q1Shuffle, _MM_SHUFFLE(0, 1, 2, 3)); + // Flip the signs on m_z and m_w + Q2Y = _mm_mul_ps(Q2Y, controlZWXY); + // Mul by Q1YXWZ + Q2Z = _mm_mul_ps(Q2Z, Q1Shuffle); + vResult = _mm_add_ps(vResult, Q2X); + // Flip the signs on m_x and m_w + Q2Z = _mm_mul_ps(Q2Z, controlYXWZ); + Q2Y = _mm_add_ps(Q2Y, Q2Z); + vResult = _mm_add_ps(vResult, Q2Y); + + return Quaternion(vResult); + } + + inline Quaternion& Quaternion::operator*=(const Quaternion& rhs) + { + *this = *this * rhs; + return *this; + } + + inline bool Quaternion::IsNearEqual(const Quaternion& rhs, Radians const threshold) const + { + return Quaternion::Distance(*this, rhs) <= threshold; + } + + inline bool Quaternion::operator==(const Quaternion& rhs) const + { + return ToVector() == rhs.ToVector(); + } + + inline bool Quaternion::operator!=(const Quaternion& rhs) const + { + return !operator==(rhs); + } + + inline Vector Quaternion::GetSplatW() const + { + return _mm_shuffle_ps(m_data, m_data, _MM_SHUFFLE(3, 3, 3, 3)); + } + + inline float Quaternion::GetW() const + { + auto vTemp = GetSplatW(); + return _mm_cvtss_f32(vTemp); + } +} diff --git a/MotionCorrection/src/cpp/Math/SIMD.h b/MotionCorrection/src/cpp/Math/SIMD.h new file mode 100644 index 0000000000000000000000000000000000000000..dfc1ced6d5f39871efcc9caf024293d8609e2825 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/SIMD.h @@ -0,0 +1,100 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +namespace SIMD +{ + struct alignas( 16 ) IntMask + { + inline operator __m128( ) const { return reinterpret_cast<__m128 const&>( *this ); } + inline operator __m128i( ) const { return _mm_castps_si128( *this ); } + inline operator __m128d( ) const { return _mm_castps_pd( *this ); } + + int32_t i[4]; + }; + + struct alignas( 16 ) UIntMask + { + inline operator __m128( ) const { return reinterpret_cast<__m128 const&>( *this ); } + inline operator __m128i( ) const { return _mm_castps_si128( *this ); } + inline operator __m128d( ) const { return _mm_castps_pd( *this ); } + + uint32_t v[4]; + }; + + struct alignas( 16 ) FloatMask + { + inline operator __m128() const { return reinterpret_cast<__m128 const&>( *this ); } + inline operator __m128i() const { return _mm_castps_si128( *this ); } + inline operator __m128d() const { return _mm_castps_pd( *this ); } + + float v[4]; + }; + + // Int Operations + //------------------------------------------------------------------------- + + namespace Int + { + FORCE_INLINE bool Equal( __m128 V1, __m128 V2 ) + { + __m128i vTemp = _mm_cmpeq_epi32( _mm_castps_si128( V1 ), _mm_castps_si128( V2 ) ); + return ( ( ( _mm_movemask_ps( _mm_castsi128_ps( vTemp ) ) & 7 ) == 7 ) != 0 ); + } + + FORCE_INLINE bool NotEqual( __m128 V1, __m128 V2 ) + { + __m128i vTemp = _mm_cmpeq_epi32( _mm_castps_si128( V1 ), _mm_castps_si128( V2 ) ); + return ( ( _mm_movemask_ps( _mm_castsi128_ps( vTemp ) ) != 0xF ) != 0 ); + } + + FORCE_INLINE __m128 And( __m128 V1, __m128 V2 ) + { + return _mm_and_ps( V1, V2 ); + } + + FORCE_INLINE __m128 Or( __m128 V1, __m128 V2 ) + { + __m128i V = _mm_or_si128( _mm_castps_si128( V1 ), _mm_castps_si128( V2 ) ); + return _mm_castsi128_ps( V ); + } + } + + //------------------------------------------------------------------------- + + static __m128 const g_sinCoefficients0 = { -0.16666667f, +0.0083333310f, -0.00019840874f, +2.7525562e-06f }; + static __m128 const g_sinCoefficients1 = { -2.3889859e-08f, -0.16665852f, +0.0083139502f, -0.00018524670f }; + static __m128 const g_cosCoefficients0 = { -0.5f, +0.041666638f, -0.0013888378f, +2.4760495e-05f }; + static __m128 const g_cosCoefficients1 = { -2.6051615e-07f, -0.49992746f, +0.041493919f, -0.0012712436f }; + static __m128 const g_tanCoefficients0 = { 1.0f, 0.333333333f, 0.133333333f, 5.396825397e-2f }; + static __m128 const g_tanCoefficients1 = { 2.186948854e-2f, 8.863235530e-3f, 3.592128167e-3f, 1.455834485e-3f }; + static __m128 const g_tanCoefficients2 = { 5.900274264e-4f, 2.391290764e-4f, 9.691537707e-5f, 3.927832950e-5f }; + static __m128 const g_arcCoefficients0 = { +1.5707963050f, -0.2145988016f, +0.0889789874f, -0.0501743046f }; + static __m128 const g_arcCoefficients1 = { +0.0308918810f, -0.0170881256f, +0.0066700901f, -0.0012624911f }; + static __m128 const g_aTanCoefficients0 = { -0.3333314528f, +0.1999355085f, -0.1420889944f, +0.1065626393f }; + static __m128 const g_aTanCoefficients1 = { -0.0752896400f, +0.0429096138f, -0.0161657367f, +0.0028662257f }; + static __m128 const g_aTanEstCoefficients0 = { +0.999866f, +0.999866f, +0.999866f, +0.999866f }; + static __m128 const g_aTanEstCoefficients1 = { -0.3302995f, +0.180141f, -0.085133f, +0.0208351f }; + static __m128 const g_tanEstCoefficients = { 2.484f, -1.954923183e-1f, 2.467401101f, Math::OneDivPi }; + static __m128 const g_arcEstCoefficients = { +1.5707288f,-0.2121144f,+0.0742610f,-0.0187293f }; + static __m128 const g_aTan2Constants = { Math::Pi, Math::PiDivTwo, Math::PiDivFour, 2.3561944905f /* 3/4 Pi */ }; + + //------------------------------------------------------------------------- + + static FloatMask const g_noFraction = { 8388608.0f,8388608.0f,8388608.0f,8388608.0f }; + static IntMask const g_absMask = { 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF, 0x7FFFFFFF }; + static UIntMask const g_trueMask = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF }; + static UIntMask const g_signMask = { 0x80000000, 0x80000000, 0x80000000, 0x80000000 }; + static UIntMask const g_maskX000 = { 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000 }; + static UIntMask const g_mask0Y00 = { 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000 }; + static UIntMask const g_mask00Z0 = { 0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000 }; + static UIntMask const g_mask000W = { 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF }; + static UIntMask const g_maskXY00 = { 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000 }; + static UIntMask const g_maskXYZ0 = { 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000 }; +} diff --git a/MotionCorrection/src/cpp/Math/Scalar.h b/MotionCorrection/src/cpp/Math/Scalar.h new file mode 100644 index 0000000000000000000000000000000000000000..425b759a3416832c3a14fb28bd85c3a4964a3525 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Scalar.h @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Compiler.h" +#include "Debug.h" + +#include "Constants.h" + +#include +#include + +// +// Scalar related methods +// + +namespace Math +{ + FORCE_INLINE float Sin( float value ) { return sinf( value ); } + FORCE_INLINE float Cos( float value ) { return cosf( value ); } + FORCE_INLINE float Tan( float value ) { return tanf( value ); } + + FORCE_INLINE float ASin( float value ) { return asinf( value ); } + FORCE_INLINE float ACos( float value ) { return acosf( value ); } + FORCE_INLINE float ATan( float value ) { return atanf( value ); } + FORCE_INLINE float ATan2( float y, float x ) { return atan2f( y, x ); } + + FORCE_INLINE float Cosec( float value ) { return 1.0f / sinf( value ); } + FORCE_INLINE float Sec( float value ) { return 1.0f / cosf( value ); } + FORCE_INLINE float Cot( float value ) { return 1.0f / tanf( PiDivTwo - value ); } + + FORCE_INLINE float Pow( float x, float y ) { return powf( x, y ); } + FORCE_INLINE float Sqr( float value ) { return value * value; } + FORCE_INLINE float Sqrt( float value ) { return sqrtf( value ); } + + FORCE_INLINE float Log( float value ) { return logf( value ); } + FORCE_INLINE float Log2f( float value ) { return log2f( value ); } + + FORCE_INLINE float AddToMovingAverage( float currentAverage, uint64_t numCurrentSamples, float newValue ) + { + return currentAverage + ( ( newValue - currentAverage ) / float( numCurrentSamples + 1 ) ); + } + + FORCE_INLINE float Abs( float a ) { return fabsf( a ); } + FORCE_INLINE double Abs( double a ) { return fabs( a ); } + FORCE_INLINE int8_t Abs( int8_t a ) { return (int8_t) abs( a ); } + FORCE_INLINE int16_t Abs( int16_t a ) { return (int16_t) abs( a ); } + FORCE_INLINE int32_t Abs( int32_t a ) { return labs( a ); } + FORCE_INLINE int64_t Abs( int64_t a ) { return llabs( a ); } + + FORCE_INLINE float Reciprocal( float r ) { return 1.0f / r; } + FORCE_INLINE double Reciprocal( double r ) { return 1.0 / r; } + + template + FORCE_INLINE T Min( T a, T b ) { return a <= b ? a : b; } + + template + FORCE_INLINE T Max( T a, T b ) { return a >= b ? a : b; } + + template + FORCE_INLINE T AbsMin( T a, T b ) { return Abs( a ) <= Abs( b ) ? a : b; } + + template + FORCE_INLINE T AbsMax( T a, T b ) { return Abs( a ) >= Abs( b ) ? a : b; } + + template + FORCE_INLINE T Sqrt( T a ) { return sqrt( a ); } + + template + FORCE_INLINE T Clamp( T value, T lowerBound, T upperBound ) + { + ASSERT( lowerBound <= upperBound ); + return Min( Max( value, lowerBound ), upperBound ); + } + + template + FORCE_INLINE bool IsInRangeInclusive( T value, T lowerBound, T upperBound ) + { + ASSERT( lowerBound < upperBound ); + return value >= lowerBound && value <= upperBound; + } + + template + FORCE_INLINE bool IsInRangeExclusive( T value, T lowerBound, T upperBound ) + { + ASSERT( lowerBound < upperBound ); + return value > lowerBound && value < upperBound; + } + + // Decomposes a float into integer and remainder portions, remainder is return and the integer result is stored in the integer portion + FORCE_INLINE float ModF( float value, float& integerPortion ) + { + return modff( value, &integerPortion ); + } + + // Returns the floating point remainder of x/y + FORCE_INLINE float FModF( float x, float y ) + { + return fmodf( x, y ); + } + + template + FORCE_INLINE T Lerp( T A, T B, float t ) + { + return A + ( B - A ) * t; + } + + FORCE_INLINE float PercentageThroughRange( float value, float lowerBound, float upperBound ) + { + ASSERT( lowerBound < upperBound ); + return Clamp( value, lowerBound, upperBound ) / ( upperBound - lowerBound ); + } + + FORCE_INLINE bool IsNearEqual( float value, float comparand, float epsilon = Epsilon ) + { + return fabsf( value - comparand ) <= epsilon; + } + + FORCE_INLINE bool IsNearZero( float value, float epsilon = Epsilon ) + { + return fabsf( value ) <= epsilon; + } + + FORCE_INLINE bool IsNearEqual( double value, double comparand, double epsilon = Epsilon ) + { + return fabs( value - comparand ) <= epsilon; + } + + FORCE_INLINE bool IsNearZero( double value, double epsilon = Epsilon ) + { + return fabs( value ) <= epsilon; + } + + FORCE_INLINE float Ceiling( float value ) + { + return ceilf( value ); + } + + FORCE_INLINE int32_t CeilingToInt( float value ) + { + return (int32_t) ceilf( value ); + } + + FORCE_INLINE float Floor( float value ) + { + return floorf( value ); + } + + FORCE_INLINE int32_t FloorToInt( float value ) + { + return (int32_t) floorf( value ); + } + + FORCE_INLINE float Round( float value ) + { + return roundf( value ); + } + + FORCE_INLINE int32_t RoundToInt( float value ) + { + return (int32_t) roundf( value ); + } + + inline float RemapRange( float value, float fromRangeBegin, float fromRangeEnd, float toRangeBegin, float toRangeEnd ) + { + float const fromRangeLength = fromRangeEnd - fromRangeBegin; + float const percentageThroughFromRange = Clamp( ( value - fromRangeBegin ) / fromRangeLength, 0.0f, 1.0f ); + float const toRangeLength = toRangeEnd - toRangeBegin; + float const result = toRangeBegin + ( percentageThroughFromRange * toRangeLength ); + + return result; + } + + FORCE_INLINE float Square( float value ) + { + return value * value; + } + + FORCE_INLINE float SmoothStep01( float value ) + { + value = Clamp( value, 0.0f, 1.0f ); + return value * value * ( 3.0f - 2.0f * value ); + } +} diff --git a/MotionCorrection/src/cpp/Math/Transform.cpp b/MotionCorrection/src/cpp/Math/Transform.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9216ddfe3b01c94265ecf2a22e6a3fb41531a7f0 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Transform.cpp @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "Transform.h" + +namespace Math +{ + Transform const Transform::Identity = Transform(Quaternion(0, 0, 0, 1), Vector(0, 0, 0, 1), 1.0f); + + void Transform::SanitizeScaleValue() + { + if (Math::IsNearEqual(GetScale(), 1.0f, Math::LargeEpsilon)) + { + SetScale(1.0f); + } + } +} diff --git a/MotionCorrection/src/cpp/Math/Transform.h b/MotionCorrection/src/cpp/Math/Transform.h new file mode 100644 index 0000000000000000000000000000000000000000..2ee49a5017c8a00c2e24ad6b61fdf89d9e13d718 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Transform.h @@ -0,0 +1,213 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Matrix.h" + +namespace Math +{ + // + // VQS Transform + // + + class Transform + { + public: + + static Transform const Identity; + + static Transform FromRotation(const Quaternion& rotation); + static Transform FromTranslation(const Vector& translation); + static Transform FromScale(float uniformScale); + static Transform FromTranslationAndScale(const Vector& translation, float uniformScale); + static Transform FromRotationBetweenVectors(const Vector sourceVector, const Vector targetVector); + + // Linearly interpolate between two transforms - uses NLerp for rotations + static Transform Lerp(const Transform& from, const Transform& to, float t); + + // Spherically interpolate between two transforms - uses SLerp for rotations + static Transform Slerp(const Transform& from, const Transform& to, float t); + + // Spherically interpolate between two transforms - uses FastSLerp (SLerp approximation) for rotations + static Transform FastSlerp(const Transform& from, const Transform& to, float t); + + // Calculate a delta transform that you can concatenate to the + // 'from' transform to get the 'to' transform. Properly handles the non-uniform scaling case. + static Transform Delta(const Transform& from, const Transform& to); + + // Calculates a delta transform that you can concatenate to the + // 'from' transform to get the 'to' transform (ignoring scale) + static Transform DeltaNoScale(const Transform& from, const Transform& to); + + static void DirectlySetRotation(Transform& transform, Quaternion&& rotation); + static void DirectlySetRotation(Transform& transform, const Quaternion& rotation); + static void DirectlySetTranslationScale(Transform& transform, Vector&& translationScale); + static void DirectlySetTranslationScale(Transform& transform, const Vector& translationScale); + + public: + + Transform() = default; + + explicit Transform(NoInit_t); + explicit Transform(const Matrix& m); + explicit Transform( + const Quaternion& rotation, + const Vector& translation = Vector(0, 0, 0, 0), + float scale = 1.0f); + explicit Transform(const AxisAngle& rotation); + + Matrix ToMatrix() const; + Matrix ToMatrixNoScale() const; + EulerAngles ToEulerAngles() const; + + Vector GetAxisX() const; + Vector GetAxisY() const; + Vector GetAxisZ() const; + + Vector GetRightVector() const; + Vector GetForwardVector() const; + Vector GetUpVector() const; + + bool IsIdentity() const; + bool IsRigidTransform() const; + void MakeRigidTransform(); + + // + // Inverse and Deltas + // + + // Invert this transform. + // If you want a delta transform that you can + // concatenate, then you should use the 'Delta' functions + Transform& Inverse(); + + // Get the inverse of this transform. + // If you want a delta transform that you can + // concatenate, then you should use the 'Delta' functions + Transform GetInverse() const; + + // Return the delta required to a given target + // transform (i.e., what do we need to add to reach that transform) + Transform GetDeltaToOther(const Transform& targetTransform) const; + + // Return the delta relative from a given a start + // transform (i.e., how much do we differ from it) + Transform GetDeltaFromOther(const Transform& startTransform) const; + + // + // Rotation + + const Quaternion& GetRotation() const; + void SetRotation(const Quaternion& rotation); + void AddRotation(const Quaternion& delta); + + // + // Translation + // + + // Get the translation for this transform + // NOTE: you cannot rely on the W value as that will be the scale + const Vector& GetTranslation() const; + + // Get the translation and scale for this transform + const Vector& GetTranslationAndScale() const; + + // Set the translation + void SetTranslation(const Vector& newTranslation); + + // Set the translation and scale simultaneously + void SetTranslationAndScale(const Vector& newTranslationScale); + + // Add an offset to the current translation + void AddTranslation(const Vector& translationDelta); + + // Get the translation as a homogeneous coordinates' vector (W=0) + Vector GetTranslationAsVector() const; + + // Get the translation as a homogeneous coordinates' point (W=1) + Vector GetTranslationAsPoint() const; + + // + // Scale + // + + float GetScale() const; + Vector GetScaleVector() const; + Vector GetInverseScaleVector() const; + void SetScale(float uniformScale); + bool HasScale() const; + bool HasNegativeScale() const; + + // This function will sanitize the scale values to remove any + // trailing values from scale factors i.e. 1.000000012 will be converted to 1 + // This is primarily needed in import steps where scale values + // might be sampled from curves or have multiple conversions applied resulting in variance. + void SanitizeScaleValue(); + + // + // Transformations + // + + Vector TranslateVector(const Vector& vector) const; + Vector ScaleVector(const Vector& vector) const; + Vector TransformPoint(const Vector& vector) const; + Vector TransformPointNoScale(const Vector& vector) const; + + // Rotate a vector (same as TransformVectorNoScale) + Vector RotateVector(const Vector& vector) const; + + // Rotate a vector (same as TransformVectorNoScale) + Vector TransformNormal(const Vector& vector) const; + + // Unrotate a vector (same as InverseTransformVectorNoScale) + Vector InverseRotateVector(const Vector& vector) const; + + // Invert the operation order when doing inverse transformation: first translation then rotation then scale + Vector InverseTransformPoint(const Vector& point) const; + + // Invert the operation order when doing inverse transformation: first translation then rotation + Vector InverseTransformPointNoScale(const Vector& point) const; + + // Applies scale and rotation to a vector (no translation) + Vector TransformVector(const Vector& vector) const; + + // Rotate a vector + Vector TransformVectorNoScale(const Vector& vector) const; + + // Invert the operation order when performing inverse transformation: first rotation then scale + Vector InverseTransformVector(const Vector& vector) const; + + // Unrotate a vector + Vector InverseTransformVectorNoScale(const Vector& vector) const; + + // WARNING: The results from multiplying transforms with shear or skew is ill-defined + Transform operator*(const Transform& rhs) const; + + // WARNING: The results from multiplying transforms with shear or skew is ill-defined + Transform& operator*=(const Transform& rhs); + + // + // Operators + // + + bool IsNearEqual( + const Transform& rhs, + const Radians angleThreshold = Math::DegreesToRadians, + float translationScaleThreshold = Math::Epsilon) const; + + // Exact equality + bool operator==(const Transform& rhs) const; + + bool operator!=(const Transform& rhs) const; + + private: + + Quaternion m_rotation = Quaternion(0, 0, 0, 1); + Vector m_translationScale = Vector(0, 0, 0, 1); + }; +} + +#include "Transform.inl" diff --git a/MotionCorrection/src/cpp/Math/Transform.inl b/MotionCorrection/src/cpp/Math/Transform.inl new file mode 100644 index 0000000000000000000000000000000000000000..0e81b096f5f190fe014d73d347675e69ff9058ec --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Transform.inl @@ -0,0 +1,508 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Transform.h" + +namespace Math +{ + inline Transform Transform::FromRotation(const Quaternion& rotation) + { + return Transform(rotation); + } + + inline Transform Transform::FromTranslation(const Vector& translation) + { + return Transform(Quaternion::Identity, translation); + } + + inline Transform Transform::FromScale(float uniformScale) + { + return Transform(Quaternion::Identity, Vector::Zero, uniformScale); + } + + inline Transform Transform::FromTranslationAndScale(const Vector& translation, float uniformScale) + { + return Transform(Quaternion::Identity, translation, uniformScale); + } + + inline Transform Transform::FromRotationBetweenVectors(const Vector sourceVector, const Vector targetVector) + { + return Transform(Quaternion::FromRotationBetweenNormalizedVectors(sourceVector, targetVector)); + } + + inline Transform Transform::Lerp(const Transform& from, const Transform& to, float t) + { + Quaternion const rotation = Quaternion::NLerp(Quaternion(from.m_rotation), Quaternion(to.m_rotation), t); + Vector const translationAndScale = Vector::Lerp(from.m_translationScale, to.m_translationScale, t); + + Transform lerped(NoInit); + lerped.m_rotation = rotation; + lerped.m_translationScale = translationAndScale; + return lerped; + } + + inline Transform Transform::Slerp(const Transform& from, const Transform& to, float t) + { + Quaternion const rotation = Quaternion::SLerp(Quaternion(from.m_rotation), Quaternion(to.m_rotation), t); + Vector const translationAndScale = Vector::Lerp(Vector(from.m_translationScale), Vector(to.m_translationScale), t); + + Transform lerped(NoInit); + lerped.m_rotation = rotation; + lerped.m_translationScale = translationAndScale; + return lerped; + } + + inline Transform Transform::FastSlerp(const Transform& from, const Transform& to, float t) + { + Quaternion const rotation = Quaternion::FastSLerp(Quaternion(from.m_rotation), Quaternion(to.m_rotation), t); + Vector const translationAndScale = Vector::Lerp(Vector(from.m_translationScale), Vector(to.m_translationScale), t); + + Transform lerped(NoInit); + lerped.m_rotation = rotation; + lerped.m_translationScale = translationAndScale; + return lerped; + } + + inline Transform Transform::Delta(const Transform& from, const Transform& to) + { + ASSERT(from.m_rotation.IsNormalized() && to.m_rotation.IsNormalized()); + ASSERT(!from.m_translationScale.IsW0() && !to.m_translationScale.IsW0()); + + Transform result; + + Vector const inverseScale = from.GetInverseScaleVector(); + Vector const deltaScale = to.GetScaleVector() * inverseScale; + + // If we have negative scaling, we need to use matrices to calculate the deltas + Vector const minScale = Vector::Min(from.m_translationScale.GetSplatW(), to.m_translationScale.GetSplatW()); + if (minScale.IsAnyLessThan(Vector::Zero)) + { + // Multiply the transforms using matrices to get the correct rotation and then remove the scale; + Matrix const toMtx = to.ToMatrix(); + Matrix const fromMtx = from.ToMatrix(); + Matrix resultMtx = toMtx * fromMtx.GetInverse(); + resultMtx.RemoveScaleFast(); + + // Apply back the signs from the final scale + Vector const sign = deltaScale.GetSign(); + resultMtx[0] *= sign.GetSplatX(); + resultMtx[1] *= sign.GetSplatY(); + resultMtx[2] *= sign.GetSplatZ(); + + result.m_rotation = resultMtx.GetRotation(); + ASSERT(result.m_rotation.IsNormalized()); + result.m_translationScale = Vector::Select(resultMtx.GetTranslation(), deltaScale, Vector::Select0001); + } + else + { + Quaternion const fromInverseRotation = from.m_rotation.GetInverse(); + result.m_rotation = to.m_rotation * fromInverseRotation; + + Vector const deltaTranslation = to.m_translationScale - from.m_translationScale; + Vector const translation = fromInverseRotation.RotateVector(deltaTranslation) * inverseScale; + result.m_translationScale = Vector::Select(translation, deltaScale, Vector::Select0001); + } + + return result; + } + + inline Transform Transform::DeltaNoScale(const Transform& from, const Transform& to) + { + Quaternion const inverseFromRotation = from.m_rotation.GetInverse(); + Vector const deltaTranslation = to.GetTranslation() - from.GetTranslation(); + + Transform delta; + delta.m_rotation = to.m_rotation * inverseFromRotation; + delta.m_translationScale = inverseFromRotation.RotateVector(deltaTranslation).GetWithW1(); + return delta; + } + + inline void Transform::DirectlySetRotation(Transform& transform, Quaternion&& rotation) + { + transform.m_rotation = rotation; + } + + inline void Transform::DirectlySetRotation(Transform& transform, const Quaternion& rotation) + { + transform.m_rotation = rotation; + } + + inline void Transform::DirectlySetTranslationScale(Transform& transform, Vector&& translationScale) + { + transform.m_translationScale = translationScale; + } + + inline void Transform::DirectlySetTranslationScale(Transform& transform, const Vector& translationScale) + { + transform.m_translationScale = translationScale; + } + + inline Transform::Transform(NoInit_t) + : m_rotation(NoInit) + , m_translationScale(NoInit) + { + } + + inline Transform::Transform(const Matrix& m) + { + Vector mTranslation, mScale; + m.Decompose(m_rotation, mTranslation, mScale); + ASSERT(Math::IsNearEqual(mScale.GetX(), mScale.GetY()) && Math::IsNearEqual(mScale.GetY(),mScale.GetZ())); + m_translationScale = Vector::Select(mTranslation, mScale, Vector::Select0001); + } + + inline Transform::Transform(const Quaternion& rotation, const Vector& translation, float scale) + : m_rotation(rotation) + , m_translationScale(Vector::Select(translation, Vector(scale), Vector::Select0001)) + { + } + + inline Transform::Transform(const AxisAngle& rotation) + : m_rotation(rotation) + , m_translationScale(Vector::UnitW) + { + } + + inline Matrix Transform::ToMatrix() const + { + return Matrix(m_rotation, m_translationScale.GetWithW1(), m_translationScale.GetSplatW()); + } + + inline Matrix Transform::ToMatrixNoScale() const + { + return Matrix(m_rotation, m_translationScale.GetWithW1(), Vector::One); + } + + inline EulerAngles Transform::ToEulerAngles() const + { + return m_rotation.ToEulerAngles(); + } + + inline Vector Transform::GetAxisX() const + { + return m_rotation.RotateVector(Vector::UnitX); + } + + inline Vector Transform::GetAxisY() const + { + return m_rotation.RotateVector(Vector::UnitY); + } + + inline Vector Transform::GetAxisZ() const + { + return m_rotation.RotateVector(Vector::UnitZ); + } + + inline Vector Transform::GetRightVector() const + { + return m_rotation.RotateVector(Vector::WorldRight); + } + + inline Vector Transform::GetForwardVector() const + { + return m_rotation.RotateVector(Vector::WorldForward); + } + + inline Vector Transform::GetUpVector() const + { + return m_rotation.RotateVector(Vector::WorldUp); + } + + inline bool Transform::IsIdentity() const + { + return m_rotation.IsIdentity() && m_translationScale.IsEqual4(Vector::UnitW); + } + + inline bool Transform::IsRigidTransform() const + { + return GetScale() == 1.0f; + } + + inline void Transform::MakeRigidTransform() + { + SetScale(1.0f); + } + + inline Transform& Transform::Inverse() + { + ASSERT(!m_translationScale.IsW0()); + + Quaternion const inverseRotation = m_rotation.GetInverse(); + m_rotation = inverseRotation; + + Vector const inverseScale = GetInverseScaleVector(); + Vector const inverselyScaledTranslation = inverseScale * m_translationScale.GetWithW0(); + Vector const inverselyRotatedTranslation = inverseRotation.RotateVector(inverselyScaledTranslation); + Vector const inverseTranslation = inverselyRotatedTranslation.GetNegated().SetW0(); + + m_translationScale = Vector::Select(inverseTranslation, inverseScale, Vector::Select0001); + + return *this; + } + + inline Transform Transform::GetInverse() const + { + Transform inverse = *this; + return inverse.Inverse(); + } + + inline Transform Transform::GetDeltaToOther(const Transform& targetTransform) const + { + return Transform::Delta(*this, targetTransform); + } + + inline Transform Transform::GetDeltaFromOther(const Transform& startTransform) const + { + return Transform::Delta(startTransform, *this); + } + + inline const Quaternion& Transform::GetRotation() const + { + return m_rotation; + } + + inline void Transform::SetRotation(const Quaternion& rotation) + { + ASSERT(rotation.IsNormalized()); + m_rotation = rotation; + } + + inline void Transform::AddRotation(const Quaternion& delta) + { + ASSERT(delta.IsNormalized()); + m_rotation = delta * m_rotation; + } + + inline const Vector& Transform::GetTranslation() const + { + return m_translationScale; + } + + inline const Vector& Transform::GetTranslationAndScale() const + { + return m_translationScale; + } + + inline void Transform::SetTranslation(const Vector& newTranslation) + { + m_translationScale = Vector::Select(newTranslation, m_translationScale, Vector::Select0001); + } + + inline void Transform::SetTranslationAndScale(const Vector& newTranslationScale) + { + ASSERT(newTranslationScale.GetW() != 0.0f); + m_translationScale = newTranslationScale; + } + + inline void Transform::AddTranslation(const Vector& translationDelta) + { + m_translationScale += translationDelta.GetWithW0(); + } + + inline Vector Transform::GetTranslationAsVector() const + { + return m_translationScale.GetWithW0(); + } + + inline Vector Transform::GetTranslationAsPoint() const + { + return m_translationScale.GetWithW1(); + } + + inline float Transform::GetScale() const + { + return m_translationScale.GetW(); + } + + inline Vector Transform::GetScaleVector() const + { + return m_translationScale.GetSplatW(); + } + + inline Vector Transform::GetInverseScaleVector() const + { + return m_translationScale.GetSplatW().GetInverse(); + } + + inline void Transform::SetScale(float uniformScale) + { + m_translationScale.SetW(uniformScale); + } + + inline bool Transform::HasScale() const + { + return m_translationScale.GetW() != 1.0f; + } + + inline bool Transform::HasNegativeScale() const + { + return m_translationScale.GetW() < 0.0f; + } + + inline Vector Transform::TranslateVector(const Vector& vector) const + { + return vector + m_translationScale.GetWithW0(); + } + + inline Vector Transform::ScaleVector(const Vector& vector) const + { + return vector * GetScaleVector(); + } + + inline Vector Transform::TransformPoint(const Vector& point) const + { + ASSERT(!m_translationScale.IsW0()); + Vector transformedPoint = point * m_translationScale.GetSplatW(); + transformedPoint = (m_translationScale + m_rotation.RotateVector(transformedPoint)).GetWithW0(); + return transformedPoint; + } + + inline Vector Transform::TransformPointNoScale(const Vector& point) const + { + Vector transformedPoint = (m_translationScale + m_rotation.RotateVector(point)).GetWithW0();; + return transformedPoint; + } + + inline Vector Transform::RotateVector(const Vector& vector) const + { + return m_rotation.RotateVector(vector); + } + + inline Vector Transform::TransformNormal(const Vector& vector) const + { + return RotateVector(vector); + } + + inline Vector Transform::InverseRotateVector(const Vector& vector) const + { + return m_rotation.RotateVectorInverse(vector); + } + + inline Vector Transform::InverseTransformPoint(const Vector& point) const + { + ASSERT(!m_translationScale.IsW0()); + Vector const shiftedPoint = point - m_translationScale; + Vector const unrotatedShiftedPoint = m_rotation.RotateVectorInverse(shiftedPoint); + Vector const inverseScale = GetInverseScaleVector(); + Vector const result = unrotatedShiftedPoint * inverseScale; + return result; + } + + inline Vector Transform::InverseTransformPointNoScale(const Vector& point) const + { + Vector const shiftedPoint = point - m_translationScale; + Vector const unrotatedShiftedPoint = m_rotation.RotateVectorInverse(shiftedPoint); + return unrotatedShiftedPoint; + } + + inline Vector Transform::TransformVector(const Vector& vector) const + { + ASSERT(!m_translationScale.IsW0()); + Vector transformedVector = vector * GetScaleVector(); + transformedVector = m_rotation.RotateVector(transformedVector); + return transformedVector; + } + + inline Vector Transform::TransformVectorNoScale(const Vector& vector) const + { + return RotateVector(vector); + } + + inline Vector Transform::InverseTransformVector(const Vector& vector) const + { + ASSERT(!m_translationScale.IsW0()); + Vector const unrotatedVector = m_rotation.RotateVectorInverse(vector); + Vector const inverseScale = GetInverseScaleVector(); + Vector const result = unrotatedVector * inverseScale; + return result; + } + + inline Vector Transform::InverseTransformVectorNoScale(const Vector& vector) const + { + return m_rotation.RotateVectorInverse(vector); + } + + inline Transform Transform::operator*(const Transform& rhs) const + { + Transform transform = *this; + transform *= rhs; + return transform; + } + + inline Transform& Transform::operator*=(const Transform& rhs) + { + Vector const scale = GetScaleVector(); + Vector const rhsScale = rhs.GetScaleVector(); + Vector const minScale = Vector::Min(scale, rhsScale); + Vector const finalScale = scale * rhsScale; + + if (minScale.IsAnyLessThan(Vector::Zero)) + { + // Multiply the transforms using matrices to + // get the correct rotation and then remove the scale; + Matrix const lhsMtx = ToMatrix(); + Matrix const rhsMtx = rhs.ToMatrix(); + Matrix resultMtx = lhsMtx * rhsMtx; + resultMtx.RemoveScaleFast(); + + // Apply back the signs from the final scale + Vector const sign = finalScale.GetSign(); + resultMtx[0] *= sign.GetSplatX(); + resultMtx[1] *= sign.GetSplatY(); + resultMtx[2] *= sign.GetSplatZ(); + + m_rotation = resultMtx.GetRotation(); + ASSERT(m_rotation.IsNormalized()); + m_translationScale = Vector::Select(resultMtx.GetTranslation(), finalScale, Vector::Select0001); + } + else + { + // Normal case + m_rotation = m_rotation * rhs.m_rotation; + m_rotation.Normalize(); + Vector const translation = rhs.m_rotation.RotateVector(m_translationScale * rhsScale) + rhs.m_translationScale; + m_translationScale = Vector::Select(translation, finalScale, Vector::Select0001); + } + + return *this; + } + + inline bool Transform::IsNearEqual(const Transform& rhs, const Radians angleThreshold, float translationScaleThreshold) const + { + if (!m_rotation.IsNearEqual(rhs.m_rotation, angleThreshold)) + { + return false; + } + + if (!m_translationScale.IsNearEqual4(rhs.m_translationScale, translationScaleThreshold)) + { + return false; + } + + return true; + } + + inline bool Transform::operator==(const Transform& rhs) const + { + if (m_translationScale != rhs.m_translationScale) + { + return false; + } + + if (m_rotation != rhs.m_rotation) + { + return false; + } + + return true; + } + + inline bool Transform::operator!=(const Transform& rhs) const + { + return !operator==(rhs); + } +} diff --git a/MotionCorrection/src/cpp/Math/Types.cpp b/MotionCorrection/src/cpp/Math/Types.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4d539ebc1298bef1f2a23b2708cce54e38be94c7 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Types.cpp @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "Types.h" + +Int2 const Int2::Zero = Int2( 0, 0 ); + +Int4 const Int4::Zero = Int4( 0, 0, 0, 0 ); +Int4 const Int4::MinusOne = Int4( -1, -1, -1, -1 ); + +Float2 const Float2::Zero = Float2( 0, 0 ); +Float2 const Float2::One = Float2( 1, 1 ); +Float2 const Float2::UnitX = Float2( 1, 0 ); +Float2 const Float2::UnitY = Float2( 0, 1 ); + +Float3 const Float3::Zero = Float3( 0, 0, 0 ); +Float3 const Float3::One = Float3( 1, 1, 1 ); +Float3 const Float3::UnitX = Float3( 1, 0, 0 ); +Float3 const Float3::UnitY = Float3( 0, 1, 0 ); +Float3 const Float3::UnitZ = Float3( 0, 0, 1 ); + +Float3 const Float3::WorldForward = Float3( 0, -1, 0 ); +Float3 const Float3::WorldUp = Float3( 0, 0, 1 ); +Float3 const Float3::WorldRight = Float3( -1, 0, 0 ); + +Float4 const Float4::Zero = Float4( 0, 0, 0, 0 ); +Float4 const Float4::One = Float4( 1, 1, 1, 1 ); +Float4 const Float4::UnitX = Float4( 1, 0, 0, 0 ); +Float4 const Float4::UnitY = Float4( 0, 1, 0, 0 ); +Float4 const Float4::UnitZ = Float4( 0, 0, 1, 0 ); +Float4 const Float4::UnitW = Float4( 0, 0, 0, 1 ); + +Float4 const Float4::WorldForward = Float4( 0, -1, 0, 0 ); +Float4 const Float4::WorldUp = Float4( 0, 0, 1, 0 ); +Float4 const Float4::WorldRight = Float4( -1, 0, 0, 0 ); + +Radians const Radians::Pi = Radians( Math::Pi ); +Radians const Radians::TwoPi = Radians( Math::TwoPi ); +Radians const Radians::OneDivPi = Radians( Math::OneDivPi ); +Radians const Radians::OneDivTwoPi = Radians( Math::OneDivTwoPi ); +Radians const Radians::PiDivTwo = Radians( Math::PiDivTwo ); +Radians const Radians::PiDivFour = Radians( Math::PiDivFour ); diff --git a/MotionCorrection/src/cpp/Math/Types.h b/MotionCorrection/src/cpp/Math/Types.h new file mode 100644 index 0000000000000000000000000000000000000000..d16b2ce10d609deba2be8dfd9c7e99e36928fa05 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Types.h @@ -0,0 +1,808 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Scalar.h" + +enum NoInit_t { NoInit }; +enum ZeroInit_t { ZeroInit }; +enum IdentityInit_t { IdentityInit }; + +enum class Axis : uint8_t +{ + X = 0, + Y, + Z, + NegX, + NegY, + NegZ +}; + +struct Float2; +struct Float3; +struct Float4; + +struct Int2 +{ + static Int2 const Zero; + +public: + + inline Int2() {} + inline Int2( ZeroInit_t ) : m_x( 0 ), m_y( 0 ) {} + inline Int2( Float2 const& v ); + inline explicit Int2( int32_t v ) : m_x( v ), m_y( v ) {} + inline explicit Int2( int32_t ix, int32_t iy ) : m_x( ix ), m_y( iy ) {} + + inline bool IsZero() const { return *this == Zero; } + + inline int32_t& operator[]( uint32_t i ) { return ( (int32_t*) this )[i]; } + inline int32_t const& operator[]( uint32_t i ) const { return ( (int32_t*) this )[i]; } + + inline bool operator==( Int2 const rhs ) const { return m_x == rhs.m_x && m_y == rhs.m_y; } + inline bool operator!=( Int2 const rhs ) const { return m_x != rhs.m_x || m_y != rhs.m_y; } + + inline Int2 operator+( Int2 const& rhs ) const { return Int2( m_x + rhs.m_x, m_y + rhs.m_y ); } + inline Int2 operator-( Int2 const& rhs ) const { return Int2( m_x - rhs.m_x, m_y - rhs.m_y ); } + inline Int2 operator*( Int2 const& rhs ) const { return Int2( m_x * rhs.m_x, m_y * rhs.m_y ); } + inline Int2 operator/( Int2 const& rhs ) const { return Int2( m_x / rhs.m_x, m_y / rhs.m_y ); } + + inline Int2& operator+=( int32_t const& rhs ) { m_x += rhs; m_y += rhs; return *this; } + inline Int2& operator-=( int32_t const& rhs ) { m_x -= rhs; m_y -= rhs; return *this; } + inline Int2& operator*=( int32_t const& rhs ) { m_x *= rhs; m_y *= rhs; return *this; } + inline Int2& operator/=( int32_t const& rhs ) { m_x /= rhs; m_y /= rhs; return *this; } + + // Component wise operation + inline Int2 operator+( int32_t const& rhs ) const { return Int2( m_x + rhs, m_y + rhs ); } + inline Int2 operator-( int32_t const& rhs ) const { return Int2( m_x - rhs, m_y - rhs ); } + inline Int2 operator*( int32_t const& rhs ) const { return Int2( m_x * rhs, m_y * rhs ); } + inline Int2 operator/( int32_t const& rhs ) const { return Int2( m_x / rhs, m_y / rhs ); } + + inline Int2& operator+=( Int2 const& rhs ) { m_x += rhs.m_x; m_y += rhs.m_y; return *this; } + inline Int2& operator-=( Int2 const& rhs ) { m_x -= rhs.m_x; m_y -= rhs.m_y; return *this; } + inline Int2& operator*=( Int2 const& rhs ) { m_x *= rhs.m_x; m_y *= rhs.m_y; return *this; } + inline Int2& operator/=( Int2 const& rhs ) { m_x /= rhs.m_x; m_y /= rhs.m_y; return *this; } + +public: + + int32_t m_x, m_y; +}; + +struct Int3 +{ + static Int3 const Zero; + +public: + + inline Int3() {} + inline Int3( ZeroInit_t ) : m_x( 0 ), m_y( 0 ), m_z( 0 ) {} + inline Int3( Float3 const& v ); + inline explicit Int3( int32_t v ) : m_x( v ), m_y( v ), m_z( v ) {} + inline explicit Int3( int32_t ix, int32_t iy, int32_t iz ) : m_x( ix ), m_y( iy ), m_z( iz ) {} + + inline bool IsZero() const { return *this == Zero; } + + inline int32_t& operator[]( uint32_t i ) { return ( (int32_t*) this )[i]; } + inline int32_t const& operator[]( uint32_t i ) const { return ( (int32_t*) this )[i]; } + + inline bool operator==( Int3 const rhs ) const { return m_x == rhs.m_x && m_y == rhs.m_y && m_z == rhs.m_z; } + inline bool operator!=( Int3 const rhs ) const { return m_x != rhs.m_x || m_y != rhs.m_y || m_z != rhs.m_z; } + + inline Int3 operator+( Int3 const& rhs ) const { return Int3( m_x + rhs.m_x, m_y + rhs.m_y, m_z + rhs.m_z ); } + inline Int3 operator-( Int3 const& rhs ) const { return Int3( m_x - rhs.m_x, m_y - rhs.m_y, m_z - rhs.m_z ); } + inline Int3 operator*( Int3 const& rhs ) const { return Int3( m_x * rhs.m_x, m_y * rhs.m_y, m_z * rhs.m_z ); } + inline Int3 operator/( Int3 const& rhs ) const { return Int3( m_x / rhs.m_x, m_y / rhs.m_y, m_z / rhs.m_z ); } + + inline Int3& operator+=( int32_t const& rhs ) { m_x += rhs; m_y += rhs; m_z += rhs; return *this; } + inline Int3& operator-=( int32_t const& rhs ) { m_x -= rhs; m_y -= rhs; m_z -= rhs; return *this; } + inline Int3& operator*=( int32_t const& rhs ) { m_x *= rhs; m_y *= rhs; m_z *= rhs; return *this; } + inline Int3& operator/=( int32_t const& rhs ) { m_x /= rhs; m_y /= rhs; m_z /= rhs; return *this; } + + // Component wise operation + inline Int3 operator+( int32_t const& rhs ) const { return Int3( m_x + rhs, m_y + rhs, m_z + rhs ); } + inline Int3 operator-( int32_t const& rhs ) const { return Int3( m_x - rhs, m_y - rhs, m_z - rhs ); } + inline Int3 operator*( int32_t const& rhs ) const { return Int3( m_x * rhs, m_y * rhs, m_z * rhs ); } + inline Int3 operator/( int32_t const& rhs ) const { return Int3( m_x / rhs, m_y / rhs, m_z / rhs ); } + + inline Int3& operator+=( Int3 const& rhs ) { m_x += rhs.m_x; m_y += rhs.m_y; m_z += rhs.m_z; return *this; } + inline Int3& operator-=( Int3 const& rhs ) { m_x -= rhs.m_x; m_y -= rhs.m_y; m_z -= rhs.m_z; return *this; } + inline Int3& operator*=( Int3 const& rhs ) { m_x *= rhs.m_x; m_y *= rhs.m_y; m_z *= rhs.m_z; return *this; } + inline Int3& operator/=( Int3 const& rhs ) { m_x /= rhs.m_x; m_y /= rhs.m_y; m_z /= rhs.m_z; return *this; } + +public: + + int32_t m_x, m_y, m_z; +}; + +struct Int4 +{ + static Int4 const Zero; + static Int4 const MinusOne; + +public: + + inline Int4() {} + inline Int4( ZeroInit_t ) : m_x( 0 ), m_y( 0 ), m_z( 0 ), m_w( 0 ) {} + inline explicit Int4( int32_t v ) : m_x( v ), m_y( v ), m_z( v ), m_w( v ) {} + inline explicit Int4( int32_t ix, int32_t iy, int32_t iz, int32_t iw ) : m_x( ix ), m_y( iy ), m_z( iz ), m_w( iw ) {} + + inline bool IsZero() const { return *this == Zero; } + + inline int32_t& operator[]( uint32_t i ) { return ( (int32_t*) this )[i]; } + inline int32_t const& operator[]( uint32_t i ) const { return ( (int32_t*) this )[i]; } + + inline bool operator==( Int4 const rhs ) const { return m_x == rhs.m_x && m_y == rhs.m_y && m_z == rhs.m_z && m_w == rhs.m_w; } + inline bool operator!=( Int4 const rhs ) const { return m_x != rhs.m_x || m_y != rhs.m_y || m_z != rhs.m_z || m_w != rhs.m_w; } + + inline Int4 operator+( int32_t const& rhs ) const { return Int4( m_x + rhs, m_y + rhs, m_z + rhs, m_w + rhs ); } + inline Int4 operator-( int32_t const& rhs ) const { return Int4( m_x - rhs, m_y - rhs, m_z - rhs, m_w - rhs ); } + inline Int4 operator*( int32_t const& rhs ) const { return Int4( m_x * rhs, m_y * rhs, m_z * rhs, m_w * rhs ); } + inline Int4 operator/( int32_t const& rhs ) const { return Int4( m_x / rhs, m_y / rhs, m_z / rhs, m_w / rhs ); } + + inline Int4& operator+=( int32_t const& rhs ) { m_x += rhs; m_y += rhs; m_z += rhs; m_w += rhs; return *this; } + inline Int4& operator-=( int32_t const& rhs ) { m_x -= rhs; m_y -= rhs; m_z -= rhs; m_w -= rhs; return *this; } + inline Int4& operator*=( int32_t const& rhs ) { m_x *= rhs; m_y *= rhs; m_z *= rhs; m_w *= rhs; return *this; } + inline Int4& operator/=( int32_t const& rhs ) { m_x /= rhs; m_y /= rhs; m_z /= rhs; m_w /= rhs; return *this; } + + // Component wise operation + inline Int4 operator+( Int4 const& rhs ) const { return Int4( m_x + rhs.m_x, m_y + rhs.m_y, m_z + rhs.m_z, m_w + rhs.m_w ); } + inline Int4 operator-( Int4 const& rhs ) const { return Int4( m_x - rhs.m_x, m_y - rhs.m_y, m_z - rhs.m_z, m_w - rhs.m_w ); } + inline Int4 operator*( Int4 const& rhs ) const { return Int4( m_x * rhs.m_x, m_y * rhs.m_y, m_z * rhs.m_z, m_w * rhs.m_w ); } + inline Int4 operator/( Int4 const& rhs ) const { return Int4( m_x / rhs.m_x, m_y / rhs.m_y, m_z / rhs.m_z, m_w / rhs.m_w ); } + + inline Int4& operator+=( Int4 const& rhs ) { m_x += rhs.m_x; m_y += rhs.m_y; m_z += rhs.m_z; m_w += rhs.m_w; return *this; } + inline Int4& operator-=( Int4 const& rhs ) { m_x -= rhs.m_x; m_y -= rhs.m_y; m_z -= rhs.m_z; m_w -= rhs.m_w; return *this; } + inline Int4& operator*=( Int4 const& rhs ) { m_x *= rhs.m_x; m_y *= rhs.m_y; m_z *= rhs.m_z; m_w *= rhs.m_w; return *this; } + inline Int4& operator/=( Int4 const& rhs ) { m_x /= rhs.m_x; m_y /= rhs.m_y; m_z /= rhs.m_z; m_w /= rhs.m_w; return *this; } + +public: + + int32_t m_x, m_y, m_z, m_w; +}; + +struct Float2 +{ + static Float2 const Zero; + static Float2 const One; + static Float2 const UnitX; + static Float2 const UnitY; + +public: + + inline Float2() {} + FORCE_INLINE Float2( ZeroInit_t ) : m_x( 0 ), m_y( 0 ) {} + FORCE_INLINE explicit Float2( float v ) : m_x( v ), m_y( v ) {} + FORCE_INLINE explicit Float2( float ix, float iy ) : m_x( ix ), m_y( iy ) {} + FORCE_INLINE explicit Float2( int32_t ix, int32_t iy ) : m_x( (float) ix ), m_y( (float) iy ) {} + inline explicit Float2( Int2 const& v ) : m_x( (float) v.m_x ), m_y( (float) v.m_y ) {} + inline explicit Float2( Float3 const& v ); + inline explicit Float2( Float4 const& v ); + + inline bool IsZero() const { return *this == Zero; } + + inline float& operator[]( uint32_t i ) { return ( (float*) this )[i]; } + inline float const& operator[]( uint32_t i ) const { return ( (float*) this )[i]; } + + FORCE_INLINE Float2 operator-() const { return Float2( -m_x, -m_y ); } + + inline bool operator==( Float2 const rhs ) const { return m_x == rhs.m_x && m_y == rhs.m_y; } + inline bool operator!=( Float2 const rhs ) const { return m_x != rhs.m_x || m_y != rhs.m_y; } + + inline Float2 operator+( Float2 const& rhs ) const { return Float2( m_x + rhs.m_x, m_y + rhs.m_y ); } + inline Float2 operator-( Float2 const& rhs ) const { return Float2( m_x - rhs.m_x, m_y - rhs.m_y ); } + inline Float2 operator*( Float2 const& rhs ) const { return Float2( m_x * rhs.m_x, m_y * rhs.m_y ); } + inline Float2 operator/( Float2 const& rhs ) const { return Float2( m_x / rhs.m_x, m_y / rhs.m_y ); } + + inline Float2 operator+( float const& rhs ) const { return Float2( m_x + rhs, m_y + rhs ); } + inline Float2 operator-( float const& rhs ) const { return Float2( m_x - rhs, m_y - rhs ); } + inline Float2 operator*( float const& rhs ) const { return Float2( m_x * rhs, m_y * rhs ); } + inline Float2 operator/( float const& rhs ) const { return Float2( m_x / rhs, m_y / rhs ); } + + inline Float2& operator+=( Float2 const& rhs ) { m_x += rhs.m_x; m_y += rhs.m_y; return *this; } + inline Float2& operator-=( Float2 const& rhs ) { m_x -= rhs.m_x; m_y -= rhs.m_y; return *this; } + inline Float2& operator*=( Float2 const& rhs ) { m_x *= rhs.m_x; m_y *= rhs.m_y; return *this; } + inline Float2& operator/=( Float2 const& rhs ) { m_x /= rhs.m_x; m_y /= rhs.m_y; return *this; } + + inline Float2& operator+=( float const& rhs ) { m_x += rhs; m_y += rhs; return *this; } + inline Float2& operator-=( float const& rhs ) { m_x -= rhs; m_y -= rhs; return *this; } + inline Float2& operator*=( float const& rhs ) { m_x *= rhs; m_y *= rhs; return *this; } + inline Float2& operator/=( float const& rhs ) { m_x /= rhs; m_y /= rhs; return *this; } + + float m_x, m_y; +}; + +struct Float3 +{ + static Float3 const Zero; + static Float3 const One; + static Float3 const UnitX; + static Float3 const UnitY; + static Float3 const UnitZ; + + static Float3 const WorldForward; + static Float3 const WorldUp; + static Float3 const WorldRight; + +public: + + inline Float3() {} + FORCE_INLINE Float3( ZeroInit_t ) : m_x( 0 ), m_y( 0 ), m_z( 0 ) {} + FORCE_INLINE explicit Float3( float v ) : m_x( v ), m_y( v ), m_z( v ) {} + FORCE_INLINE explicit Float3( float ix, float iy, float iz ) : m_x( ix ), m_y( iy ), m_z( iz ) {} + inline explicit Float3( Float2 const& v, float iz = 0.0f ) : m_x( v.m_x ), m_y( v.m_y ), m_z( iz ) {} + inline explicit Float3( Float4 const& v ); + + inline bool IsZero() const { return *this == Zero; } + + inline float& operator[]( uint32_t i ) { return ( (float*) this )[i]; } + inline float const& operator[]( uint32_t i ) const { return ( (float*) this )[i]; } + + FORCE_INLINE Float3 operator-() const { return Float3( -m_x, -m_y, -m_z ); } + + inline bool operator==( Float3 const rhs ) const { return m_x == rhs.m_x && m_y == rhs.m_y && m_z == rhs.m_z; } + inline bool operator!=( Float3 const rhs ) const { return m_x != rhs.m_x || m_y != rhs.m_y || m_z != rhs.m_z; } + + inline operator Float2() const { return Float2( m_x, m_y ); } + + inline Float3 operator+( Float3 const& rhs ) const { return Float3( m_x + rhs.m_x, m_y + rhs.m_y, m_z + rhs.m_z ); } + inline Float3 operator-( Float3 const& rhs ) const { return Float3( m_x - rhs.m_x, m_y - rhs.m_y, m_z - rhs.m_z ); } + inline Float3 operator*( Float3 const& rhs ) const { return Float3( m_x * rhs.m_x, m_y * rhs.m_y, m_z * rhs.m_z ); } + inline Float3 operator/( Float3 const& rhs ) const { return Float3( m_x / rhs.m_x, m_y / rhs.m_y, m_z / rhs.m_z ); } + + inline Float3 operator+( float const& rhs ) const { return Float3( m_x + rhs, m_y + rhs, m_z + rhs ); } + inline Float3 operator-( float const& rhs ) const { return Float3( m_x - rhs, m_y - rhs, m_z - rhs ); } + inline Float3 operator*( float const& rhs ) const { return Float3( m_x * rhs, m_y * rhs, m_z * rhs ); } + inline Float3 operator/( float const& rhs ) const { return Float3( m_x / rhs, m_y / rhs, m_z / rhs ); } + + inline Float3& operator+=( Float3 const& rhs ) { m_x += rhs.m_x; m_y += rhs.m_y; m_z += rhs.m_z; return *this; } + inline Float3& operator-=( Float3 const& rhs ) { m_x -= rhs.m_x; m_y -= rhs.m_y; m_z -= rhs.m_z; return *this; } + inline Float3& operator*=( Float3 const& rhs ) { m_x *= rhs.m_x; m_y *= rhs.m_y; m_z *= rhs.m_z; return *this; } + inline Float3& operator/=( Float3 const& rhs ) { m_x /= rhs.m_x; m_y /= rhs.m_y; m_z /= rhs.m_z; return *this; } + + inline Float3& operator+=( float const& rhs ) { m_x += rhs; m_y += rhs; m_z += rhs; return *this; } + inline Float3& operator-=( float const& rhs ) { m_x -= rhs; m_y -= rhs; m_z -= rhs; return *this; } + inline Float3& operator*=( float const& rhs ) { m_x *= rhs; m_y *= rhs; m_z *= rhs; return *this; } + inline Float3& operator/=( float const& rhs ) { m_x /= rhs; m_y /= rhs; m_z /= rhs; return *this; } + + float m_x, m_y, m_z; +}; + +struct Float4 +{ + static Float4 const Zero; + static Float4 const One; + static Float4 const UnitX; + static Float4 const UnitY; + static Float4 const UnitZ; + static Float4 const UnitW; + + static Float4 const WorldForward; + static Float4 const WorldUp; + static Float4 const WorldRight; + +public: + + Float4() {} + FORCE_INLINE Float4( ZeroInit_t ) : m_x( 0 ), m_y( 0 ), m_z( 0 ), m_w( 0 ) {} + FORCE_INLINE explicit Float4( float v ) : m_x( v ), m_y( v ), m_z( v ), m_w( v ) {} + FORCE_INLINE explicit Float4( float ix, float iy, float iz, float iw ) : m_x( ix ), m_y( iy ), m_z( iz ), m_w( iw ) {} + explicit Float4( Float2 const& v, float iz = 0.0f, float iw = 0.0f ) : m_x( v.m_x ), m_y( v.m_y ), m_z( iz ), m_w( iw ) {} + explicit Float4( Float3 const& v, float iw = 0.0f ) : m_x( v.m_x ), m_y( v.m_y ), m_z( v.m_z ), m_w( iw ) {} + + inline bool IsZero() const { return *this == Zero; } + + float& operator[]( uint32_t i ) { return ( (float*) this )[i]; } + float const& operator[]( uint32_t i ) const { return ( (float*) this )[i]; } + + FORCE_INLINE Float4 operator-() const { return Float4( -m_x, -m_y, -m_z, -m_w ); } + + bool operator==( Float4 const rhs ) const { return m_x == rhs.m_x && m_y == rhs.m_y && m_z == rhs.m_z && m_w == rhs.m_w; } + bool operator!=( Float4 const rhs ) const { return m_x != rhs.m_x || m_y != rhs.m_y || m_z != rhs.m_z || m_w != rhs.m_w; } + + inline operator Float2() const { return Float2( m_x, m_y ); } + inline operator Float3() const { return Float3( m_x, m_y, m_z ); } + + inline Float4 operator+( Float4 const& rhs ) const { return Float4( m_x + rhs.m_x, m_y + rhs.m_y, m_z + rhs.m_z, m_w + rhs.m_w ); } + inline Float4 operator-( Float4 const& rhs ) const { return Float4( m_x - rhs.m_x, m_y - rhs.m_y, m_z - rhs.m_z, m_w - rhs.m_w ); } + inline Float4 operator*( Float4 const& rhs ) const { return Float4( m_x * rhs.m_x, m_y * rhs.m_y, m_z * rhs.m_z, m_w * rhs.m_w ); } + inline Float4 operator/( Float4 const& rhs ) const { return Float4( m_x / rhs.m_x, m_y / rhs.m_y, m_z / rhs.m_z, m_w / rhs.m_w ); } + + inline Float4 operator+( float const& rhs ) const { return Float4( m_x + rhs, m_y + rhs, m_z + rhs, m_w + rhs ); } + inline Float4 operator-( float const& rhs ) const { return Float4( m_x - rhs, m_y - rhs, m_z - rhs, m_w - rhs ); } + inline Float4 operator*( float const& rhs ) const { return Float4( m_x * rhs, m_y * rhs, m_z * rhs, m_w * rhs ); } + inline Float4 operator/( float const& rhs ) const { return Float4( m_x / rhs, m_y / rhs, m_z / rhs, m_w / rhs ); } + + inline Float4& operator+=( Float4 const& rhs ) { m_x += rhs.m_x; m_y += rhs.m_y; m_z += rhs.m_z; m_w += rhs.m_w; return *this; } + inline Float4& operator-=( Float4 const& rhs ) { m_x -= rhs.m_x; m_y -= rhs.m_y; m_z -= rhs.m_z; m_w -= rhs.m_w; return *this; } + inline Float4& operator*=( Float4 const& rhs ) { m_x *= rhs.m_x; m_y *= rhs.m_y; m_z *= rhs.m_z; m_w *= rhs.m_w; return *this; } + inline Float4& operator/=( Float4 const& rhs ) { m_x /= rhs.m_x; m_y /= rhs.m_y; m_z /= rhs.m_z; m_w /= rhs.m_w; return *this; } + + inline Float4& operator+=( float const& rhs ) { m_x += rhs; m_y += rhs; m_z += rhs; m_w += rhs; return *this; } + inline Float4& operator-=( float const& rhs ) { m_x -= rhs; m_y -= rhs; m_z -= rhs; m_w -= rhs; return *this; } + inline Float4& operator*=( float const& rhs ) { m_x *= rhs; m_y *= rhs; m_z *= rhs; m_w *= rhs; return *this; } + inline Float4& operator/=( float const& rhs ) { m_x /= rhs; m_y /= rhs; m_z /= rhs; m_w /= rhs; return *this; } + + float m_x, m_y, m_z, m_w; +}; + +inline Int2::Int2( Float2 const& v ) + : m_x( (int32_t) v.m_x ) + , m_y( (int32_t) v.m_y ) +{ +} + +inline Int3::Int3( Float3 const& v ) + : m_x( (int32_t) v.m_x ) + , m_y( (int32_t) v.m_y ) + , m_z( (int32_t) v.m_z ) +{ +} + +inline Float2::Float2( Float3 const& v ) + : m_x( v.m_x ) + , m_y( v.m_y ) +{ +} + +inline Float2::Float2( Float4 const& v ) + : m_x( v.m_x ) + , m_y( v.m_y ) +{ +} + +inline Float3::Float3( Float4 const& v ) + : m_x( v.m_x ) + , m_y( v.m_y ) + , m_z( v.m_z ) +{ +} + +struct Radians; +struct Degrees; + +struct Degrees +{ +public: + + inline Degrees() = default; + inline Degrees( float degrees ) : m_value( degrees ) {} + inline explicit Degrees( Radians const& radians ); + + FORCE_INLINE explicit operator float() const { return m_value; } + FORCE_INLINE operator Radians() const; + FORCE_INLINE float ToFloat() const { return m_value; } + FORCE_INLINE Radians ToRadians() const; + + inline Degrees operator-() const { return Degrees( -m_value ); } + + inline Degrees operator+( Degrees const& rhs ) const { return Degrees( m_value + rhs.m_value ); } + inline Degrees operator-( Degrees const& rhs ) const { return Degrees( m_value - rhs.m_value ); } + inline Degrees operator*( Degrees const& rhs ) const { return Degrees( m_value * rhs.m_value ); } + inline Degrees operator/( Degrees const& rhs ) const { return Degrees( m_value / rhs.m_value ); } + + inline Degrees& operator+=( Degrees const& rhs ) { m_value += rhs.m_value; return *this; } + inline Degrees& operator-=( Degrees const& rhs ) { m_value -= rhs.m_value; return *this; } + inline Degrees& operator*=( Degrees const& rhs ) { m_value *= rhs.m_value; return *this; } + inline Degrees& operator/=( Degrees const& rhs ) { m_value /= rhs.m_value; return *this; } + + inline Degrees operator+( float const& rhs ) const { return Degrees( m_value + rhs ); } + inline Degrees operator-( float const& rhs ) const { return Degrees( m_value - rhs ); } + inline Degrees operator*( float const& rhs ) const { return Degrees( m_value * rhs ); } + inline Degrees operator/( float const& rhs ) const { return Degrees( m_value / rhs ); } + + inline Degrees& operator+=( float const& rhs ) { m_value += rhs; return *this; } + inline Degrees& operator-=( float const& rhs ) { m_value -= rhs; return *this; } + inline Degrees& operator*=( float const& rhs ) { m_value *= rhs; return *this; } + inline Degrees& operator/=( float const& rhs ) { m_value /= rhs; return *this; } + + inline Degrees operator+( int32_t const& rhs ) const { return Degrees( m_value + rhs ); } + inline Degrees operator-( int32_t const& rhs ) const { return Degrees( m_value - rhs ); } + inline Degrees operator*( int32_t const& rhs ) const { return Degrees( m_value * rhs ); } + inline Degrees operator/( int32_t const& rhs ) const { return Degrees( m_value / rhs ); } + + inline Degrees& operator+=( int32_t const& rhs ) { m_value += rhs; return *this; } + inline Degrees& operator-=( int32_t const& rhs ) { m_value -= rhs; return *this; } + inline Degrees& operator*=( int32_t const& rhs ) { m_value *= rhs; return *this; } + inline Degrees& operator/=( int32_t const& rhs ) { m_value /= rhs; return *this; } + + inline Degrees operator+( uint32_t const& rhs ) const { return Degrees( m_value + rhs ); } + inline Degrees operator-( uint32_t const& rhs ) const { return Degrees( m_value - rhs ); } + inline Degrees operator*( uint32_t const& rhs ) const { return Degrees( m_value * rhs ); } + inline Degrees operator/( uint32_t const& rhs ) const { return Degrees( m_value / rhs ); } + + inline Degrees& operator+=( uint32_t const& rhs ) { m_value += rhs; return *this; } + inline Degrees& operator-=( uint32_t const& rhs ) { m_value -= rhs; return *this; } + inline Degrees& operator*=( uint32_t const& rhs ) { m_value *= rhs; return *this; } + inline Degrees& operator/=( uint32_t const& rhs ) { m_value /= rhs; return *this; } + + inline bool operator>( float const& rhs ) const { return m_value > rhs; }; + inline bool operator<( float const& rhs ) const { return m_value < rhs; } + inline bool operator>=( float const& rhs ) const { return m_value >= rhs; } + inline bool operator<=( float const& rhs ) const { return m_value <= rhs; } + + inline bool operator>( Degrees const& rhs ) const { return m_value > rhs.m_value; } + inline bool operator<( Degrees const& rhs ) const { return m_value < rhs.m_value; } + inline bool operator>=( Degrees const& rhs ) const { return m_value >= rhs.m_value; } + inline bool operator<=( Degrees const& rhs ) const { return m_value <= rhs.m_value; } + + inline bool operator>( Radians const& rhs ) const; + inline bool operator<( Radians const& rhs ) const; + inline bool operator>=( Radians const& rhs ) const; + inline bool operator<=( Radians const& rhs ) const; + + inline bool operator==( float const& v ) const { return Math::IsNearEqual( m_value, v ); } + inline bool operator!=( float const& v ) const { return !Math::IsNearEqual( m_value, v ); } + + inline bool operator==( Degrees const& rhs ) const { return m_value == rhs.m_value; } + inline bool operator!=( Degrees const& rhs ) const { return m_value != rhs.m_value; } + + inline bool operator==( Radians const& rhs ) const; + inline bool operator!=( Radians const& rhs ) const; + + inline void Clamp( Degrees min, Degrees max ) + { + m_value = Math::Clamp( m_value, min.m_value, max.m_value ); + } + + // Clamps between -360 and 360 + inline void Clamp360() + { + m_value -= ( int32_t( m_value / 360.0f ) * 360.0f ); + } + + // Clamps between -360 and 360 + inline Degrees GetClamped360() const + { + Degrees d( m_value ); + d.Clamp360(); + return d; + } + + // Clamps to -180 to 180 + inline void Clamp180() + { + Clamp360(); + + float delta = 180 - Math::Abs( m_value ); + if ( delta < 0 ) + { + delta += 180; + m_value = ( m_value < 0 ) ? delta : -delta; + } + } + + // Clamps to -180 to 180 + inline Degrees GetClamped180() const + { + Degrees r( m_value ); + r.Clamp180(); + return r; + } + + // Clamps between 0 to 360 + inline Degrees& ClampPositive360() + { + Clamp360(); + if ( m_value < 0 ) + { + m_value += 360; + } + return *this; + } + + // Clamps between 0 to 360 + inline Degrees GetClampedPositive360() const + { + Degrees d( m_value ); + d.ClampPositive360(); + return d; + } + +private: + + float m_value = 0; +}; + +struct Radians +{ + static Radians const Pi; + static Radians const TwoPi; + static Radians const OneDivPi; + static Radians const OneDivTwoPi; + static Radians const PiDivTwo; + static Radians const PiDivFour; + +public: + + inline Radians() = default; + inline Radians( float radians ) : m_value( radians ) {} + inline explicit Radians( Degrees const& degrees ); + + FORCE_INLINE explicit operator float() const { return m_value; } + FORCE_INLINE operator Degrees() const { return ToDegrees(); } + FORCE_INLINE float ToFloat() const { return m_value; } + FORCE_INLINE Degrees ToDegrees() const { return Degrees( m_value * Math::RadiansToDegrees ); } + + inline Radians operator-() const { return Radians( -m_value ); } + + inline Radians operator+( Radians const& rhs ) const { return Radians( m_value + rhs.m_value ); } + inline Radians operator-( Radians const& rhs ) const { return Radians( m_value - rhs.m_value ); } + inline Radians operator*( Radians const& rhs ) const { return Radians( m_value * rhs.m_value ); } + inline Radians operator/( Radians const& rhs ) const { return Radians( m_value / rhs.m_value ); } + + inline Radians& operator+=( Radians const& rhs ) { m_value += rhs.m_value; return *this; } + inline Radians& operator-=( Radians const& rhs ) { m_value -= rhs.m_value; return *this; } + inline Radians& operator*=( Radians const& rhs ) { m_value *= rhs.m_value; return *this; } + inline Radians& operator/=( Radians const& rhs ) { m_value /= rhs.m_value; return *this; } + + inline Radians operator+( float const& rhs ) const { return Radians( m_value + rhs ); } + inline Radians operator-( float const& rhs ) const { return Radians( m_value - rhs ); } + inline Radians operator*( float const& rhs ) const { return Radians( m_value * rhs ); } + inline Radians operator/( float const& rhs ) const { return Radians( m_value / rhs ); } + + inline Radians& operator+=( float const& rhs ) { m_value += rhs; return *this; } + inline Radians& operator-=( float const& rhs ) { m_value -= rhs; return *this; } + inline Radians& operator*=( float const& rhs ) { m_value *= rhs; return *this; } + inline Radians& operator/=( float const& rhs ) { m_value /= rhs; return *this; } + + inline Radians operator+( int32_t const& rhs ) const { return Radians( m_value + rhs ); } + inline Radians operator-( int32_t const& rhs ) const { return Radians( m_value - rhs ); } + inline Radians operator*( int32_t const& rhs ) const { return Radians( m_value * rhs ); } + inline Radians operator/( int32_t const& rhs ) const { return Radians( m_value / rhs ); } + + inline Radians& operator+=( int32_t const& rhs ) { m_value += rhs; return *this; } + inline Radians& operator-=( int32_t const& rhs ) { m_value -= rhs; return *this; } + inline Radians& operator*=( int32_t const& rhs ) { m_value *= rhs; return *this; } + inline Radians& operator/=( int32_t const& rhs ) { m_value /= rhs; return *this; } + + inline Radians operator+( uint32_t const& rhs ) const { return Radians( m_value + rhs ); } + inline Radians operator-( uint32_t const& rhs ) const { return Radians( m_value - rhs ); } + inline Radians operator*( uint32_t const& rhs ) const { return Radians( m_value * rhs ); } + inline Radians operator/( uint32_t const& rhs ) const { return Radians( m_value / rhs ); } + + inline Radians& operator+=( uint32_t const& rhs ) { m_value += rhs; return *this; } + inline Radians& operator-=( uint32_t const& rhs ) { m_value -= rhs; return *this; } + inline Radians& operator*=( uint32_t const& rhs ) { m_value *= rhs; return *this; } + inline Radians& operator/=( uint32_t const& rhs ) { m_value /= rhs; return *this; } + + inline bool operator>( float const& rhs ) const { return m_value > rhs; }; + inline bool operator<( float const& rhs ) const { return m_value < rhs; } + inline bool operator>=( float const& rhs ) const { return m_value >= rhs; } + inline bool operator<=( float const& rhs ) const { return m_value <= rhs; } + + inline bool operator>( Radians const& rhs ) const { return m_value > rhs.m_value; } + inline bool operator<( Radians const& rhs ) const { return m_value < rhs.m_value; } + inline bool operator>=( Radians const& rhs ) const { return m_value >= rhs.m_value; } + inline bool operator<=( Radians const& rhs ) const { return m_value <= rhs.m_value; } + + inline bool operator>( Degrees const& rhs ) const; + inline bool operator<( Degrees const& rhs ) const; + inline bool operator>=( Degrees const& rhs ) const; + inline bool operator<=( Degrees const& rhs ) const; + + inline bool operator==( float const& v ) const { return Math::IsNearEqual( m_value, v ); } + inline bool operator!=( float const& v ) const { return !Math::IsNearEqual( m_value, v ); } + + inline bool operator==( Radians const& rhs ) const { return m_value == rhs.m_value; } + inline bool operator!=( Radians const& rhs ) const { return m_value != rhs.m_value; } + + inline bool operator==( Degrees const& rhs ) const; + inline bool operator!=( Degrees const& rhs ) const; + + inline void Clamp( Radians min, Radians max ) + { + m_value = Math::Clamp( m_value, min.m_value, max.m_value ); + } + + // Clamps between -2Pi to 2Pi + inline void Clamp360() + { + m_value -= int32_t( m_value / Math::TwoPi ) * Math::TwoPi; + } + + // Clamps between -2Pi to 2Pi + inline Radians GetClamped360() const + { + Radians r( m_value ); + r.Clamp360(); + return r; + } + + // Clamps between 0 to 2Pi + inline void ClampPositive360() + { + Clamp360(); + if( m_value < 0 ) + { + m_value += Math::TwoPi; + } + } + + // Clamps between 0 to 2Pi + inline Radians GetClampedToPositive360() const + { + Radians r( m_value ); + r.ClampPositive360(); + return r; + } + + // Clamps to -Pi to Pi + inline void Clamp180() + { + Clamp360(); + + float delta = Math::Pi - Math::Abs( m_value ); + if ( delta < 0 ) + { + delta += Math::Pi; + m_value = ( m_value < 0 ) ? delta : -delta; + } + } + + // Clamps to -Pi to Pi + inline Radians GetClamped180() const + { + Radians r( m_value ); + r.Clamp180(); + return r; + } + + // Inverts angle between [0;2Pi] and [-2Pi;0] + inline void Invert() + { + Clamp360(); + float const delta = Math::TwoPi - Math::Abs( m_value ); + m_value = ( m_value < 0 ) ? delta : -delta; + } + + // Inverts angle between [0;2Pi] and [-2Pi;0] + inline Radians GetInverse() const + { + Radians r( m_value ); + r.Invert(); + return r; + } + + // Flips the front and rear 180 degree arc i.e. 135 becomes -45, -90 becomes 90, etc. + inline void Flip() + { + Clamp180(); + float const delta = Math::Pi - Math::Abs( m_value ); + m_value = ( m_value < 0 ) ? delta : -delta; + } + + // Flips the front and rear 180 degree arc i.e. 135 becomes -45, -90 becomes 90, etc. + inline Radians GetFlipped() const + { + Radians r( m_value ); + r.Flip(); + return r; + } + +private: + + float m_value = 0; +}; + +inline Degrees::Degrees( Radians const& radians ) + : m_value( radians.ToDegrees() ) +{} + +inline Radians Degrees::ToRadians() const +{ + return Radians( m_value * Math::DegreesToRadians ); +} + +inline Degrees::operator Radians() const +{ + return ToRadians(); +} + +inline bool Degrees::operator>( Radians const& rhs ) const { return m_value > rhs.ToDegrees().m_value; } +inline bool Degrees::operator<( Radians const& rhs ) const { return m_value < rhs.ToDegrees().m_value; } +inline bool Degrees::operator>=( Radians const& rhs ) const { return m_value >= rhs.ToDegrees().m_value; } +inline bool Degrees::operator<=( Radians const& rhs ) const { return m_value <= rhs.ToDegrees().m_value; } + +inline bool Degrees::operator==( Radians const& rhs ) const { return Math::IsNearEqual( m_value, rhs.ToDegrees().m_value ); } +inline bool Degrees::operator!=( Radians const& rhs ) const { return !Math::IsNearEqual( m_value, rhs.ToDegrees().m_value ); } + +inline Radians::Radians( Degrees const& degrees ) + : m_value( degrees.ToRadians() ) +{} + +inline bool Radians::operator>( Degrees const& rhs ) const { return m_value > rhs.ToRadians().m_value; } +inline bool Radians::operator<( Degrees const& rhs ) const { return m_value < rhs.ToRadians().m_value; } +inline bool Radians::operator>=( Degrees const& rhs ) const { return m_value >= rhs.ToRadians().m_value; } +inline bool Radians::operator<=( Degrees const& rhs ) const { return m_value <= rhs.ToRadians().m_value; } + +inline bool Radians::operator==( Degrees const& rhs ) const { return Math::IsNearEqual( m_value, rhs.ToRadians().m_value ); } +inline bool Radians::operator!=( Degrees const& rhs ) const { return !Math::IsNearEqual( m_value, rhs.ToRadians().m_value ); } + +struct EulerAngles +{ +public: + + EulerAngles() = default; + + inline explicit EulerAngles( Degrees inX, Degrees inY, Degrees inZ ) + : m_x( inX ) + , m_y( inY ) + , m_z( inZ ) + {} + + inline explicit EulerAngles( Radians inX, Radians inY, Radians inZ ) + : m_x( inX ) + , m_y( inY ) + , m_z( inZ ) + {} + + inline explicit EulerAngles( float inDegreesX, float inDegreesY, float inDegreesZ ) + : m_x( Math::DegreesToRadians * inDegreesX ) + , m_y( Math::DegreesToRadians * inDegreesY ) + , m_z( Math::DegreesToRadians * inDegreesZ ) + {} + + inline EulerAngles( Float3 const& anglesInDegrees ) + : m_x( Math::DegreesToRadians * anglesInDegrees.m_x ) + , m_y( Math::DegreesToRadians * anglesInDegrees.m_y ) + , m_z( Math::DegreesToRadians * anglesInDegrees.m_z ) + {} + + inline void Clamp() + { + m_x.Clamp360(); + m_y.Clamp360(); + m_z.Clamp360(); + } + + inline EulerAngles GetClamped() const + { + EulerAngles clamped = *this; + clamped.Clamp(); + return clamped; + } + + inline Radians GetYaw() const { return m_z; } + inline Radians GetPitch() const { return m_x; } + inline Radians GetRoll() const { return m_y; } + + inline Float3 GetAsRadians() const { return Float3( m_x.ToFloat(), m_y.ToFloat(), m_z.ToFloat() ); } + inline Float3 GetAsDegrees() const { return Float3( m_x.ToDegrees().ToFloat(), m_y.ToDegrees().ToFloat(), m_z.ToDegrees().ToFloat() ); } + + inline bool operator==( EulerAngles const& other ) const { return m_x == other.m_x && m_y == other.m_y && m_z == other.m_z; } + inline bool operator!=( EulerAngles const& other ) const { return m_x != other.m_x || m_y != other.m_y || m_z != other.m_z; } + + inline Radians& operator[]( uint32_t i ) { return ( (Radians*) this )[i]; } + inline Radians const& operator[]( uint32_t i ) const { return ( (Radians*) this )[i]; } + // in degrees + inline Float3 ToFloat3() const { return Float3( Math::RadiansToDegrees * m_x.ToFloat(), Math::RadiansToDegrees * m_y.ToFloat(), Math::RadiansToDegrees * m_z.ToFloat() ); } + +public: + + Radians m_x = 0.0f; + Radians m_y = 0.0f; + Radians m_z = 0.0f; +}; + +struct AxisAngle +{ +public: + + inline AxisAngle() = default; + inline explicit AxisAngle( Float3 axis, Radians angle ) : m_axis( axis ), m_angle( angle ) {} + inline explicit AxisAngle( Float3 axis, Degrees angle ) : m_axis( axis ), m_angle( angle.ToRadians() ) {} + + inline bool IsValid() const + { + float const lengthSq = m_axis.m_x * m_axis.m_x + m_axis.m_y * m_axis.m_y + m_axis.m_z * m_axis.m_z; + return Math::Abs( lengthSq - 1.0f ) < Math::Epsilon; + } + +public: + + Float3 m_axis = Float3::Zero; + Radians m_angle = Radians( 0.0f ); +}; diff --git a/MotionCorrection/src/cpp/Math/Vector.cpp b/MotionCorrection/src/cpp/Math/Vector.cpp new file mode 100644 index 0000000000000000000000000000000000000000..965adc9aa5ea3ed588c26347c338535b5e263815 --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Vector.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "Vector.h" +#include "Quaternion.h" + +namespace Math +{ + Vector const Vector::UnitX = { 1, 0, 0, 0 }; + Vector const Vector::UnitY = { 0, 1, 0, 0 }; + Vector const Vector::UnitZ = { 0, 0, 1, 0 }; + Vector const Vector::UnitW = { 0, 0, 0, 1 }; + + Vector const Vector::Origin = { 0, 0, 0, 1 }; + Vector const Vector::WorldForward = { 0, -1, 0, 0 }; + Vector const Vector::WorldBackward = { 0, 1, 0, 0 }; + Vector const Vector::WorldUp = { 0, 0, 1, 0 }; + Vector const Vector::WorldDown = { 0, 0, -1, 0 }; + Vector const Vector::WorldLeft = { 1, 0, 0, 0 }; + Vector const Vector::WorldRight = { -1, 0, 0, 0 }; + + Vector const Vector::Infinity = { 0x7F800000, 0x7F800000, 0x7F800000, 0x7F800000 }; + Vector const Vector::QNaN = { 0x7FC00000, 0x7FC00000, 0x7FC00000, 0x7FC00000 }; + + Vector const Vector::NegativeOne(-1.0f); + Vector const Vector::Zero(0.0f); + Vector const Vector::Half(0.5f); + Vector const Vector::One(1.0f); + + Vector const Vector::Epsilon(Math::Epsilon); + Vector const Vector::LargeEpsilon(Math::LargeEpsilon); + Vector const Vector::OneMinusEpsilon(1.0f - Math::Epsilon); + Vector const Vector::EpsilonMinusOne(Math::Epsilon - 1.0f); + Vector const Vector::NormalizeCheckThreshold(0.01f); // Squared Error + + Vector const Vector::Pi(Math::Pi); + Vector const Vector::PiDivTwo(Math::PiDivTwo); + Vector const Vector::TwoPi(Math::TwoPi); + Vector const Vector::OneDivTwoPi(Math::OneDivTwoPi); + + Vector const Vector::Select0000(0, 0, 0, 0); + Vector const Vector::Select0001(0, 0, 0, 1); + Vector const Vector::Select0010(0, 0, 1, 0); + Vector const Vector::Select0011(0, 0, 1, 1); + Vector const Vector::Select0100(0, 1, 0, 0); + Vector const Vector::Select0101(0, 1, 0, 1); + Vector const Vector::Select0110(0, 1, 1, 0); + Vector const Vector::Select0111(0, 1, 1, 1); + Vector const Vector::Select1000(1, 0, 0, 0); + Vector const Vector::Select1001(1, 0, 0, 1); + Vector const Vector::Select1010(1, 0, 1, 0); + Vector const Vector::Select1011(1, 0, 1, 1); + Vector const Vector::Select1100(1, 1, 0, 0); + Vector const Vector::Select1101(1, 1, 0, 1); + Vector const Vector::Select1110(1, 1, 1, 0); + Vector const Vector::Select1111(1, 1, 1, 1); + + Vector const Vector::BoxCorners[8] = + { + { -1.0f, -1.0f, 1.0f, 0.0f }, + { 1.0f, -1.0f, 1.0f, 0.0f }, + { 1.0f, 1.0f, 1.0f, 0.0f }, + { -1.0f, 1.0f, 1.0f, 0.0f }, + { -1.0f, -1.0f, -1.0f, 0.0f }, + { 1.0f, -1.0f, -1.0f, 0.0f }, + { 1.0f, 1.0f, -1.0f, 0.0f }, + { -1.0f, 1.0f, -1.0f, 0.0f }, + }; + + Vector Vector::SLerp(const Vector& from, const Vector& to, float t) + { + ASSERT(t >= 0.0f && t <= 1.0f); + if (from.LengthSquared3().IsLessThan4(Epsilon) || to.LengthSquared3().IsLessThan4(Epsilon)) + { + return Lerp(from, to, t); + } + + // Calculate the final length + const Vector fromLength = from.Length3(); + const Vector toLength = to.Length3(); + const Vector finalLength = Lerp(fromLength, toLength, t); + + // Normalize vectors + const Vector normalizedFrom = from / fromLength; + const Vector normalizedTo = to / toLength; + + // Handle parallel vector + Vector result; + if (normalizedFrom.IsParallelTo(normalizedTo)) + { + result = normalizedFrom; + } + else + { + // Interpolate the rotation between the vectors + const Vector dot = Dot3(normalizedFrom, normalizedTo); + const Vector angle = ACos(dot); + const Vector axis = Cross3(normalizedFrom, normalizedTo).Normalize3(); + const Vector interpolatedAngle = Lerp(Zero, angle, t); + + const Quaternion rotation(axis, Radians(interpolatedAngle.ToFloat())); + const Vector finalDirection = rotation.RotateVector(normalizedFrom); + result = finalDirection.GetNormalized3() * finalLength; + } + + return result; + } +} diff --git a/MotionCorrection/src/cpp/Math/Vector.h b/MotionCorrection/src/cpp/Math/Vector.h new file mode 100644 index 0000000000000000000000000000000000000000..4d19bea3b6c215aa20ba636e5347b45a36812c8c --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Vector.h @@ -0,0 +1,454 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "Compiler.h" + +#include "Types.h" +#include "Constants.h" +#include "SIMD.h" + +namespace Math +{ + class alignas(16) Vector + { + public: + + static Vector const UnitX; + static Vector const UnitY; + static Vector const UnitZ; + static Vector const UnitW; + + static Vector const Origin; + static Vector const WorldForward; + static Vector const WorldBackward; + static Vector const WorldUp; + static Vector const WorldDown; + static Vector const WorldLeft; + static Vector const WorldRight; + + static Vector const NegativeOne; + static Vector const Zero; + static Vector const Half; + static Vector const One; + static Vector const Epsilon; + static Vector const LargeEpsilon; + static Vector const OneMinusEpsilon; + static Vector const EpsilonMinusOne; + static Vector const NormalizeCheckThreshold; + static Vector const Pi; + static Vector const PiDivTwo; + static Vector const TwoPi; + static Vector const OneDivTwoPi; + + static Vector const Select0000; + static Vector const Select0001; + static Vector const Select0010; + static Vector const Select0011; + static Vector const Select0100; + static Vector const Select0101; + static Vector const Select0110; + static Vector const Select0111; + static Vector const Select1000; + static Vector const Select1001; + static Vector const Select1010; + static Vector const Select1011; + static Vector const Select1100; + static Vector const Select1101; + static Vector const Select1110; + static Vector const Select1111; + + static Vector const Infinity; + static Vector const QNaN; + + static Vector const BoxCorners[8]; + + // + // Utils + // + + static Vector Cross2(const Vector& v0, const Vector& v1); + static Vector Cross3(const Vector& v0, const Vector& v1); + static Vector Dot2(const Vector& v0, const Vector& v1); + static Vector Dot3(const Vector& v0, const Vector& v1); + static Vector Dot4(const Vector& v0, const Vector& v1); + static Vector Average2(const Vector& v0, const Vector& v1); + static Vector Average3(const Vector& v0, const Vector& v1); + static Vector Average4(const Vector& v0, const Vector& v1); + static Vector Min(const Vector& v0, const Vector& v1); + static Vector Max(const Vector& v0, const Vector& v1); + static float Min(const Vector& v); + static float Max(const Vector& v); + static Vector Clamp(const Vector& v, const Vector& min, const Vector& max); + static Vector Xor(const Vector& vec0, const Vector& vec1); + + // Add the multiplied results to a vector: ( vec * mul ) + addend + static Vector MultiplyAdd(const Vector& vec, const Vector& multiplier, const Vector& addend); + + // Subtract a vector from the multiplied result: (vec * mul ) - subtrahend + static Vector MultiplySubtract(const Vector& vec, const Vector& multiplier, const Vector& subtrahend); + + // Subtract the multiplied result from a vector: minuend - (vec * mul ) + static Vector NegativeMultiplySubtract(const Vector& vec, const Vector& multiplier, const Vector& minuend); + + // Sum up scaled versions of two vectors + static Vector LinearCombination(const Vector& v0, const Vector& v1, float scale0, float scale1); + + // Linear interpolation of one vector to another + static Vector Lerp(const Vector& from, const Vector& to, float t); + + // Normalized linear interpolation of one vector to another + static Vector NLerp(const Vector& from, const Vector& to, float t); + + // Spherical interpolation of one vector to another + static Vector SLerp(const Vector& from, const Vector& to, float t); + + // Combine the two vectors based on the control: 0 means select from v0, 1 means select from v1. E.G. To select XY from v0 and ZW from v1, control = Vector( 0, 0, 1, 1 ) + static Vector Select(const Vector& v0, const Vector& v1, const Vector& control); + + // Get a permutation of two vectors, each template argument represents the element index to select ( v0: 0-3, v1: 4-7 ); + template + static Vector Permute(const Vector& v0, const Vector& v1); + + // + // Trigonometry + // + + static Vector Sin(const Vector& vec); + static Vector Cos(const Vector& vec); + static Vector Tan(const Vector& vec); + static Vector ASin(const Vector& vec); + static Vector ACos(const Vector& vec); + static Vector ATan(const Vector& vec); + static Vector ATan2(const Vector& vec0, const Vector& vec1); + + static Vector SinEst(const Vector& vec); + static Vector CosEst(const Vector& vec); + static Vector TanEst(const Vector& vec); + static Vector ASinEst(const Vector& vec); + static Vector ACosEst(const Vector& vec); + static Vector ATanEst(const Vector& vec); + static Vector ATan2Est(const Vector& vec0, const Vector& vec1); + + static void SinCos(Vector& sin, Vector& cos, float angle); + static void SinCos(Vector& sin, Vector& cos, const Vector& angle); + + static Vector AngleMod2Pi(const Vector& angles); + + public: + + operator __m128& (); + operator const __m128& () const; + + Vector(); + explicit Vector(Axis axis); + explicit Vector(ZeroInit_t); + explicit Vector(float v); + Vector(__m128 v); + Vector(float ix, float iy, float iz, float iw = 1.0f); + + Vector(const Float2& v, float iz = 0.0f, float iw = 0.0f); + Vector(const Float3& v, float iw = 1.0f); + Vector(const Float4& v); + Vector(const float* pValues); + + bool IsValid() const; + + void Store(float* pValues) const; + void StoreFloat(float& value) const; + void StoreFloat2(Float2& value) const; + void StoreFloat3(Float3& value) const; + void StoreFloat4(Float4& value) const; + + float ToFloat() const; + Float2 ToFloat2() const; + Float3 ToFloat3() const; + Float4 ToFloat4() const; + + operator Float2() const; + operator Float3() const; + operator Float4() const; + + // + // Element accessors + // + + float GetX() const; + float GetY() const; + float GetZ() const; + float GetW() const; + + void SetX(float x); + void SetY(float y); + void SetZ(float z); + void SetW(float w); + + float operator[](uint32_t i) const; + + // + // W component operations + // + + bool IsW1() const; + bool IsW0() const; + Vector& SetW0(); + Vector& SetW1(); + Vector GetWithW0() const; + Vector GetWithW1() const; + + // + // Dimensional Getters + // + + // Returns only the first two components, z=w=0 + Vector Get2D() const; + + // Returns only the first three components, w = 0 + Vector Get3D() const; + + // + // Algebraic operators + // + + Vector operator+(const Vector& v) const; + Vector& operator+=(const Vector& v); + Vector operator-(const Vector& v) const; + Vector& operator-=(const Vector& v); + Vector operator*(const Vector& v) const; + Vector& operator*=(const Vector& v); + Vector operator/(const Vector& v) const; + Vector& operator/=(const Vector& v); + + Vector operator*(float const f) const; + Vector& operator*=(float const f); + Vector operator/(float const f) const; + Vector& operator/=(float const f); + + Vector operator-() const; + + Vector Orthogonal2D() const; + Vector Cross2(const Vector& other) const; + Vector Cross3(const Vector& other) const; + Vector Dot2(const Vector& other) const; + Vector Dot3(const Vector& other) const; + Vector Dot4(const Vector& other) const; + float GetDot2(const Vector& other) const; + float GetDot3(const Vector& other) const; + float GetDot4(const Vector& other) const; + + Vector ScalarProjection(const Vector& other) const; + float GetScalarProjection(const Vector& other) const; + Vector VectorProjection(const Vector& other) const; + + // + // Transformations + // + + Vector& Invert(); + Vector GetInverse() const; + Vector GetReciprocal() const; + + Vector& InvertEst(); + Vector GetInverseEst() const; + + Vector& Negate(); + Vector GetNegated() const; + + Vector& Abs(); + Vector GetAbs() const; + + Vector& Sqrt(); + Vector GetSqrt(); + + Vector& ReciprocalSqrt(); + Vector GetReciprocalSqrt(); + + Vector& EstimatedReciprocalSqrt(); + Vector GetEstimatedReciprocalSqrt(); + + Vector& Normalize2(); + Vector& Normalize3(); + Vector& Normalize4(); + + Vector GetNormalized2() const; + Vector GetNormalized3() const; + Vector GetNormalized4() const; + + Vector& Floor(); + Vector GetFloor() const; + Vector& Ceil(); + Vector GetCeil() const; + Vector& Round(); + Vector GetRound() const; + + Vector GetSign() const; + + // + // Permutations + // + + Vector GetSplatX() const; + Vector GetSplatY() const; + Vector GetSplatZ() const; + Vector GetSplatW() const; + + // Get a shuffled version of this vector, each argument represents the element index in the original vector + template + Vector Swizzle() const; + + // Get a shuffled version of this vector, each argument represents the element index in the original vector + Vector Swizzle(uint32_t xIdx, uint32_t yIdx, uint32_t zIdx, uint32_t wIdx) const; + + // Get a shuffled version of this vector, each argument represents the element index in the original vector + Vector Shuffle(uint32_t xIdx, uint32_t yIdx, uint32_t zIdx, uint32_t wIdx) const; + + // Get a shuffled version of this vector, each argument represents the element index in the original vector + template + Vector Shuffle() const; + + // + // Queries + // + + Vector Length2() const; + Vector Length3() const; + Vector Length4() const; + + float GetLength2() const; + float GetLength3() const; + float GetLength4() const; + + Vector InverseLength2() const; + Vector InverseLength3() const; + Vector InverseLength4() const; + + float GetInverseLength2() const; + float GetInverseLength3() const; + float GetInverseLength4() const; + + Vector LengthSquared2() const; + Vector LengthSquared3() const; + Vector LengthSquared4() const; + + float GetLengthSquared2() const; + float GetLengthSquared3() const; + float GetLengthSquared4() const; + + Vector Distance2(const Vector& to) const; + Vector Distance3(const Vector& to) const; + Vector Distance4(const Vector& to) const; + + float GetDistance2(const Vector& to) const; + float GetDistance3(const Vector& to) const; + float GetDistance4(const Vector& to) const; + + Vector DistanceSquared2(const Vector& to) const; + Vector DistanceSquared3(const Vector& to) const; + Vector DistanceSquared4(const Vector& to) const; + + float GetDistanceSquared2(const Vector& to) const; + float GetDistanceSquared3(const Vector& to) const; + float GetDistanceSquared4(const Vector& to) const; + + bool IsNormalized2() const; + bool IsNormalized3() const; + bool IsNormalized4() const; + + // Is this vector within the range [-bounds, bounds] + Vector InBounds(const Vector& bounds) const; + + bool IsInBounds2(const Vector& bounds) const; + bool IsInBounds3(const Vector& bounds) const; + bool IsInBounds4(const Vector& bounds) const; + + Vector Equal(const Vector& v) const; + + bool IsEqual2(const Vector& v) const; + bool IsEqual3(const Vector& v) const; + bool IsEqual4(const Vector& v) const; + + Vector NearEqual(const Vector& v, const Vector& epsilon) const; + + bool IsNearEqual2(const Vector& v, float epsilon) const; + bool IsNearEqual3(const Vector& v, float epsilon) const; + bool IsNearEqual4(const Vector& v, float epsilon) const; + + bool IsNearEqual2(const Vector& v, const Vector& epsilon = Vector::Epsilon) const; + bool IsNearEqual3(const Vector& v, const Vector& epsilon = Vector::Epsilon) const; + bool IsNearEqual4(const Vector& v, const Vector& epsilon = Vector::Epsilon) const; + + Vector GreaterThan(const Vector& v) const; + bool IsAnyGreaterThan(const Vector& v) const; + + bool IsGreaterThan2(const Vector& v) const; + bool IsGreaterThan3(const Vector& v) const; + bool IsGreaterThan4(const Vector& v) const; + + Vector GreaterThanEqual(const Vector& v) const; + bool IsAnyGreaterThanEqual(const Vector& v) const; + + bool IsGreaterThanEqual2(const Vector& v) const; + bool IsGreaterThanEqual3(const Vector& v) const; + bool IsGreaterThanEqual4(const Vector& v) const; + + Vector LessThan(const Vector& v) const; + bool IsAnyLessThan(const Vector& v) const; + + bool IsLessThan2(const Vector& v) const; + bool IsLessThan3(const Vector& v) const; + bool IsLessThan4(const Vector& v) const; + + Vector LessThanEqual(const Vector& v) const; + bool IsAnyLessThanEqual(const Vector& v) const; + + bool IsLessThanEqual2(const Vector& v) const; + bool IsLessThanEqual3(const Vector& v) const; + bool IsLessThanEqual4(const Vector& v) const; + + Vector EqualsZero() const; + bool IsAnyEqualToZero2() const; + bool IsAnyEqualToZero3() const; + bool IsAnyEqualToZero4() const; + + bool IsZero2() const; + bool IsZero3() const; + bool IsZero4() const; + + Vector NearEqualsZero(float epsilon = Math::Epsilon) const; + + bool IsNearZero2(float epsilon = Math::Epsilon) const; + bool IsNearZero3(float epsilon = Math::Epsilon) const; + bool IsNearZero4(float epsilon = Math::Epsilon) const; + + Vector EqualsInfinity() const; + + bool IsInfinite2() const; + bool IsInfinite3() const; + bool IsInfinite4() const; + + Vector EqualsNaN() const; + + bool IsNaN2() const; + bool IsNaN3() const; + bool IsNaN4() const; + + bool IsParallelTo(const Vector& v) const; + + void ToDirectionAndLength2(Vector& direction, float& length) const; + void ToDirectionAndLength3(Vector& direction, float& length) const; + + bool operator==(const Vector& rhs) const; + bool operator!=(const Vector& rhs) const; + + public: + + __m128 m_data; + }; + + static_assert(sizeof(Vector) == 16, "Vector size must be 16 bytes!"); +} + +#include "Vector.inl" diff --git a/MotionCorrection/src/cpp/Math/Vector.inl b/MotionCorrection/src/cpp/Math/Vector.inl new file mode 100644 index 0000000000000000000000000000000000000000..14e4169b3b68cc3be40f42ddfcc466f90fd80e7e --- /dev/null +++ b/MotionCorrection/src/cpp/Math/Vector.inl @@ -0,0 +1,2224 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "Vector.h" + +namespace Math +{ + FORCE_INLINE Vector Vector::Cross2(const Vector& v0, const Vector& v1) + { + return v0.Cross2(v1); + } + + FORCE_INLINE Vector Vector::Cross3(const Vector& v0, const Vector& v1) + { + return v0.Cross3(v1); + } + + FORCE_INLINE Vector Vector::Dot2(const Vector& v0, const Vector& v1) + { + return v0.Dot2(v1); + } + + FORCE_INLINE Vector Vector::Dot3(const Vector& v0, const Vector& v1) + { + return v0.Dot3(v1); + } + + FORCE_INLINE Vector Vector::Dot4(const Vector& v0, const Vector& v1) + { + return v0.Dot4(v1); + } + + FORCE_INLINE Vector Vector::Average2(const Vector& v0, const Vector& v1) + { + auto avg4 = Average4(v0, v1); + return Vector::Select(avg4, Vector::Zero, Vector(0, 0, 1, 1)); + } + + FORCE_INLINE Vector Vector::Average3(const Vector& v0, const Vector& v1) + { + auto avg4 = Average4(v0, v1); + return Vector::Select(avg4, Vector::Zero, Vector(0, 0, 0, 1)); + } + + FORCE_INLINE Vector Vector::Average4(const Vector& v0, const Vector& v1) + { + return (v0 + v1) * Vector::Half; + } + + FORCE_INLINE Vector Vector::Min(const Vector& v0, const Vector& v1) + { + Vector result; + result = _mm_min_ps(v0, v1); + return result; + } + + FORCE_INLINE Vector Vector::Max(const Vector& v0, const Vector& v1) + { + Vector result; + result = _mm_max_ps(v0, v1); + return result; + } + + FORCE_INLINE float Vector::Min(const Vector& v) + { + __m128 shufReg, sumsReg; + shufReg = _mm_movehdup_ps(v); + sumsReg = _mm_min_ps(v, shufReg); + shufReg = _mm_movehl_ps(shufReg, sumsReg); + sumsReg = _mm_min_ss(sumsReg, shufReg); + return _mm_cvtss_f32(sumsReg); + } + + FORCE_INLINE float Vector::Max(const Vector& v) + { + __m128 shufReg, sumsReg; + shufReg = _mm_movehdup_ps(v); + sumsReg = _mm_max_ps(v, shufReg); + shufReg = _mm_movehl_ps(shufReg, sumsReg); + sumsReg = _mm_max_ss(sumsReg, shufReg); + return _mm_cvtss_f32(sumsReg); + } + + FORCE_INLINE Vector Vector::Clamp(const Vector& v, const Vector& min, const Vector& max) + { + Vector result; + result = _mm_max_ps(min, v); + result = _mm_min_ps(result, max); + return result; + } + + FORCE_INLINE Vector Vector::Xor(const Vector& v0, const Vector& v1) + { + __m128i V = _mm_xor_si128(_mm_castps_si128(v0), _mm_castps_si128(v1)); + + Vector result; + result = _mm_castsi128_ps(V); + return result; + } + + FORCE_INLINE Vector Vector::MultiplyAdd(const Vector& v, const Vector& multiplier, const Vector& addend) + { + // result = addend + ( vec * multiplier ) + Vector result; + result = _mm_mul_ps(v, multiplier); + result = _mm_add_ps(result, addend); + return result; + } + + FORCE_INLINE Vector Vector::MultiplySubtract(const Vector& vec, const Vector& multiplier, const Vector& subtrahend) + { + // result = ( vec * multiplier ) - subtrahend + auto r = _mm_mul_ps(vec, multiplier); + return _mm_sub_ps(r, subtrahend); + } + + FORCE_INLINE Vector Vector::NegativeMultiplySubtract(const Vector& vec, const Vector& multiplier, const Vector& minuend) + { + // result = minuend - ( vec * multiplier ) + auto r = _mm_mul_ps(vec, multiplier); + return _mm_sub_ps(minuend, r); + } + + FORCE_INLINE Vector Vector::LinearCombination(const Vector& v0, const Vector& v1, float scale0, float scale1) + { + return (v0 * scale0) + (v1 * scale1); + } + + FORCE_INLINE Vector Vector::Lerp(const Vector& from, const Vector& to, float t) + { + ASSERT(t >= 0.0f && t <= 1.0f); + + Vector L = _mm_sub_ps(to, from); + Vector S = _mm_set_ps1(t); + + Vector result; + result = _mm_mul_ps(L, S); + result = _mm_add_ps(result, from); + return result; + } + + FORCE_INLINE Vector Vector::NLerp(const Vector& from, const Vector& to, float t) + { + ASSERT(t >= 0.0f && t <= 1.0f); + + // Calculate the final length + auto const fromLength = from.Length3(); + auto const toLength = to.Length3(); + auto const finalLength = Vector::Lerp(fromLength, toLength, t); + + // Normalize vectors + Vector const normalizedFrom = from / fromLength; + Vector const normalizedTo = to / toLength; + + // LERP + auto const finalDirection = Lerp(normalizedFrom, normalizedTo, t); + auto result = finalDirection.GetNormalized3() * finalLength; + return result; + } + + FORCE_INLINE Vector Vector::Select(const Vector& v0, const Vector& v1, const Vector& control) + { + auto const ctrl = _mm_cmpneq_ps(control, Vector::Zero); + + Vector result; + auto vTemp1 = _mm_andnot_ps(ctrl, v0); + auto vTemp2 = _mm_and_ps(v1, ctrl); + result = _mm_or_ps(vTemp1, vTemp2); + return result; + } + + template + FORCE_INLINE Vector Vector::Permute(const Vector& v0, const Vector& v1) + { + static_assert(PermuteX <= 7, "Element index parameter out of range"); + static_assert(PermuteY <= 7, "Element index parameter out of range"); + static_assert(PermuteZ <= 7, "Element index parameter out of range"); + static_assert(PermuteW <= 7, "Element index parameter out of range"); + + uint32_t const shuffle = _MM_SHUFFLE(PermuteW & 3, PermuteZ & 3, PermuteY & 3, PermuteX & 3); + bool const whichX = PermuteX > 3; + bool const whichY = PermuteY > 3; + bool const whichZ = PermuteZ > 3; + bool const whichW = PermuteW > 3; + + static SIMD::UIntMask const selectMask = { whichX ? 0xFFFFFFFF : 0, whichY ? 0xFFFFFFFF : 0, whichZ ? 0xFFFFFFFF : 0, whichW ? 0xFFFFFFFF : 0 }; + __m128 shuffled1 = _mm_shuffle_ps(v0, v0, shuffle); + __m128 shuffled2 = _mm_shuffle_ps(v1, v1, shuffle); + __m128 masked1 = _mm_andnot_ps(selectMask, shuffled1); + __m128 masked2 = _mm_and_ps(selectMask, shuffled2); + return _mm_or_ps(masked1, masked2); + } + + FORCE_INLINE Vector Vector::Sin(const Vector& vec) + { + // Force the value within the bounds of pi + auto m_x = Vector::AngleMod2Pi(vec); + + // Map in [-pi/2,pi/2] with sin(m_y) = sin(m_x). + __m128 sign = _mm_and_ps(m_x, SIMD::g_signMask); + __m128 c = _mm_or_ps(Vector::Pi, sign); // pi when m_x >= 0, -pi when m_x < 0 + __m128 absx = _mm_andnot_ps(sign, m_x); // |m_x| + __m128 rflx = _mm_sub_ps(c, m_x); + __m128 comp = _mm_cmple_ps(absx, Vector::PiDivTwo); + __m128 select0 = _mm_and_ps(comp, m_x); + __m128 select1 = _mm_andnot_ps(comp, rflx); + m_x = _mm_or_ps(select0, select1); + + __m128 x2 = _mm_mul_ps(m_x, m_x); + + // Compute polynomial approximation + const auto SC1 = SIMD::g_sinCoefficients1; + auto vConstants = _mm_shuffle_ps(SC1, SC1, _MM_SHUFFLE(0, 0, 0, 0)); + __m128 Result = _mm_mul_ps(vConstants, x2); + + const auto SC0 = SIMD::g_sinCoefficients0; + vConstants = _mm_shuffle_ps(SC0, SC0, _MM_SHUFFLE(3, 3, 3, 3)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(SC0, SC0, _MM_SHUFFLE(2, 2, 2, 2)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(SC0, SC0, _MM_SHUFFLE(1, 1, 1, 1)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(SC0, SC0, _MM_SHUFFLE(0, 0, 0, 0)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + Result = _mm_add_ps(Result, Vector::One); + Result = _mm_mul_ps(Result, m_x); + return Result; + } + + FORCE_INLINE Vector Vector::Cos(const Vector& vec) + { + // Map V to m_x in [-pi,pi]. + auto m_x = Vector::AngleMod2Pi(vec); + + // Map in [-pi/2,pi/2] with cos(m_y) = sign*cos(m_x). + auto sign = _mm_and_ps(m_x, SIMD::g_signMask); + __m128 c = _mm_or_ps(Vector::Pi, sign); // pi when m_x >= 0, -pi when m_x < 0 + __m128 absx = _mm_andnot_ps(sign, m_x); // |m_x| + __m128 rflx = _mm_sub_ps(c, m_x); + __m128 comp = _mm_cmple_ps(absx, Vector::PiDivTwo); + __m128 select0 = _mm_and_ps(comp, m_x); + __m128 select1 = _mm_andnot_ps(comp, rflx); + m_x = _mm_or_ps(select0, select1); + select0 = _mm_and_ps(comp, Vector::One); + select1 = _mm_andnot_ps(comp, Vector::NegativeOne); + sign = _mm_or_ps(select0, select1); + + __m128 x2 = _mm_mul_ps(m_x, m_x); + + // Compute polynomial approximation + const auto CC1 = SIMD::g_cosCoefficients1; + auto vConstants = _mm_shuffle_ps(CC1, CC1, _MM_SHUFFLE(0, 0, 0, 0)); + __m128 Result = _mm_mul_ps(vConstants, x2); + + const auto CC0 = SIMD::g_cosCoefficients0; + vConstants = _mm_shuffle_ps(CC0, CC0, _MM_SHUFFLE(3, 3, 3, 3)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(CC0, CC0, _MM_SHUFFLE(2, 2, 2, 2)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(CC0, CC0, _MM_SHUFFLE(1, 1, 1, 1)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(CC0, CC0, _MM_SHUFFLE(0, 0, 0, 0)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + Result = _mm_add_ps(Result, Vector::One); + Result = _mm_mul_ps(Result, sign); + return Result; + } + + FORCE_INLINE Vector Vector::Tan(const Vector& vec) + { + static const Vector tanCoefficients0 = { 1.0f, -4.667168334e-1f, 2.566383229e-2f, -3.118153191e-4f }; + static const Vector tanCoefficients1 = { 4.981943399e-7f, -1.333835001e-1f, 3.424887824e-3f, -1.786170734e-5f }; + static const Vector tanConstants = { 1.570796371f, 6.077100628e-11f, 0.000244140625f, 0.63661977228f /*2 / Pi*/ }; + static const SIMD::UIntMask mask = { 0x1, 0x1, 0x1, 0x1 }; + + Vector TwoDivPi = tanConstants.GetSplatW(); + Vector C0 = tanConstants.GetSplatX(); + Vector C1 = tanConstants.GetSplatY(); + Vector vEpsilon = tanConstants.GetSplatZ(); + + Vector VA = (vec * TwoDivPi).Round(); + Vector VC = Vector::NegativeMultiplySubtract(VA, C0, vec); + Vector VB = VA.GetAbs(); + VC = Vector::NegativeMultiplySubtract(VA, C1, VC); + reinterpret_cast<__m128i*>(&VB)[0] = _mm_cvttps_epi32(VB); + + Vector VC2 = VC * VC; + Vector T7 = tanCoefficients1.GetSplatW(); + Vector T6 = tanCoefficients1.GetSplatZ(); + Vector T4 = tanCoefficients1.GetSplatX(); + Vector T3 = tanCoefficients0.GetSplatW(); + Vector T5 = tanCoefficients1.GetSplatY(); + Vector T2 = tanCoefficients0.GetSplatZ(); + Vector T1 = tanCoefficients0.GetSplatY(); + Vector T0 = tanCoefficients0.GetSplatX(); + + Vector VBIsEven = _mm_and_ps(VB, mask); + VBIsEven = _mm_castsi128_ps(_mm_cmpeq_epi32(_mm_castps_si128(VBIsEven), _mm_castps_si128(Vector::Zero))); + + Vector N = Vector::MultiplyAdd(VC2, T7, T6); + Vector D = Vector::MultiplyAdd(VC2, T4, T3); + N = Vector::MultiplyAdd(VC2, N, T5); + D = Vector::MultiplyAdd(VC2, D, T2); + N = VC2 * N; + D = Vector::MultiplyAdd(VC2, D, T1); + N = Vector::MultiplyAdd(VC, N, VC); + Vector VCNearZero = VC.InBounds(vEpsilon); + D = Vector::MultiplyAdd(VC2, D, T0); + + N = Vector::Select(N, VC, VCNearZero); + D = Vector::Select(D, Vector::One, VCNearZero); + + Vector R0 = N.GetNegated(); + Vector R1 = N / D; + R0 = D / R0; + + Vector VIsZero = vec.EqualsZero(); + Vector Result = Vector::Select(R0, R1, VBIsEven); + Result = Vector::Select(Result, Zero, VIsZero); + + return Result; + } + + FORCE_INLINE Vector Vector::ASin(const Vector& vec) + { + __m128 nonnegative = _mm_cmpge_ps(vec, Vector::Zero); + __m128 mvalue = _mm_sub_ps(Vector::Zero, vec); + __m128 m_x = _mm_max_ps(vec, mvalue); // |vec| + + // Compute (1-|vec|), clamp to zero to avoid sqrt of negative number. + __m128 oneMValue = _mm_sub_ps(Vector::One, m_x); + __m128 clampOneMValue = _mm_max_ps(Vector::Zero, oneMValue); + __m128 root = _mm_sqrt_ps(clampOneMValue); // sqrt(1-|vec|) + + // Compute polynomial approximation + const auto AC1 = SIMD::g_arcCoefficients1; + auto vConstants = _mm_shuffle_ps(AC1, AC1, _MM_SHUFFLE(3, 3, 3, 3)); + __m128 t0 = _mm_mul_ps(vConstants, m_x); + + vConstants = _mm_shuffle_ps(AC1, AC1, _MM_SHUFFLE(2, 2, 2, 2)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AC1, AC1, _MM_SHUFFLE(1, 1, 1, 1)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AC1, AC1, _MM_SHUFFLE(0, 0, 0, 0)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + const auto AC0 = SIMD::g_arcCoefficients0; + vConstants = _mm_shuffle_ps(AC0, AC0, _MM_SHUFFLE(3, 3, 3, 3)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AC0, AC0, _MM_SHUFFLE(2, 2, 2, 2)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AC0, AC0, _MM_SHUFFLE(1, 1, 1, 1)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AC0, AC0, _MM_SHUFFLE(0, 0, 0, 0)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, root); + + __m128 t1 = _mm_sub_ps(Vector::Pi, t0); + t0 = _mm_and_ps(nonnegative, t0); + t1 = _mm_andnot_ps(nonnegative, t1); + t0 = _mm_or_ps(t0, t1); + t0 = _mm_sub_ps(Vector::PiDivTwo, t0); + return t0; + } + + FORCE_INLINE Vector Vector::ACos(const Vector& vec) + { + __m128 nonnegative = _mm_cmpge_ps(vec, Vector::Zero); + __m128 mvalue = _mm_sub_ps(Vector::Zero, vec); + __m128 m_x = _mm_max_ps(vec, mvalue); // |vec| + + // Compute (1-|vec|), clamp to zero to avoid sqrt of negative number. + __m128 oneMValue = _mm_sub_ps(Vector::One, m_x); + __m128 clampOneMValue = _mm_max_ps(Vector::Zero, oneMValue); + __m128 root = _mm_sqrt_ps(clampOneMValue); // sqrt(1-|vec|) + + // Compute polynomial approximation + const auto AC1 = SIMD::g_arcCoefficients1; + auto vConstants = _mm_shuffle_ps(AC1, AC1, _MM_SHUFFLE(3, 3, 3, 3)); + __m128 t0 = _mm_mul_ps(vConstants, m_x); + + vConstants = _mm_shuffle_ps(AC1, AC1, _MM_SHUFFLE(2, 2, 2, 2)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AC1, AC1, _MM_SHUFFLE(1, 1, 1, 1)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AC1, AC1, _MM_SHUFFLE(0, 0, 0, 0)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + const auto AC0 = SIMD::g_arcCoefficients0; + vConstants = _mm_shuffle_ps(AC0, AC0, _MM_SHUFFLE(3, 3, 3, 3)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AC0, AC0, _MM_SHUFFLE(2, 2, 2, 2)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AC0, AC0, _MM_SHUFFLE(1, 1, 1, 1)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AC0, AC0, _MM_SHUFFLE(0, 0, 0, 0)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, root); + + __m128 t1 = _mm_sub_ps(Vector::Pi, t0); + t0 = _mm_and_ps(nonnegative, t0); + t1 = _mm_andnot_ps(nonnegative, t1); + t0 = _mm_or_ps(t0, t1); + return t0; + } + + FORCE_INLINE Vector Vector::ATan(const Vector& vec) + { + __m128 absV = vec.GetAbs(); + __m128 invV = _mm_div_ps(Vector::One, vec); + __m128 comp = _mm_cmpgt_ps(vec, Vector::One); + __m128 select0 = _mm_and_ps(comp, Vector::One); + __m128 select1 = _mm_andnot_ps(comp, Vector::NegativeOne); + __m128 sign = _mm_or_ps(select0, select1); + comp = _mm_cmple_ps(absV, Vector::One); + select0 = _mm_and_ps(comp, Vector::Zero); + select1 = _mm_andnot_ps(comp, sign); + sign = _mm_or_ps(select0, select1); + select0 = _mm_and_ps(comp, vec); + select1 = _mm_andnot_ps(comp, invV); + __m128 m_x = _mm_or_ps(select0, select1); + + __m128 x2 = _mm_mul_ps(m_x, m_x); + + // Compute polynomial approximation + Vector const TC1 = SIMD::g_aTanCoefficients1; + Vector vConstants = _mm_shuffle_ps(TC1, TC1, _MM_SHUFFLE(3, 3, 3, 3)); + __m128 Result = _mm_mul_ps(vConstants, x2); + + vConstants = _mm_shuffle_ps(TC1, TC1, _MM_SHUFFLE(2, 2, 2, 2)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(TC1, TC1, _MM_SHUFFLE(1, 1, 1, 1)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(TC1, TC1, _MM_SHUFFLE(0, 0, 0, 0)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + Vector const TC0 = SIMD::g_aTanCoefficients0; + vConstants = _mm_shuffle_ps(TC0, TC0, _MM_SHUFFLE(3, 3, 3, 3)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(TC0, TC0, _MM_SHUFFLE(2, 2, 2, 2)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(TC0, TC0, _MM_SHUFFLE(1, 1, 1, 1)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(TC0, TC0, _MM_SHUFFLE(0, 0, 0, 0)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + Result = _mm_add_ps(Result, Vector::One); + Result = _mm_mul_ps(Result, m_x); + __m128 result1 = _mm_mul_ps(sign, Vector::PiDivTwo); + result1 = _mm_sub_ps(result1, Result); + + comp = _mm_cmpeq_ps(sign, Vector::Zero); + select0 = _mm_and_ps(comp, Result); + select1 = _mm_andnot_ps(comp, result1); + Result = _mm_or_ps(select0, select1); + return Result; + } + + FORCE_INLINE Vector Vector::ATan2(const Vector& Y, const Vector& X) + { + Vector ATanResultValid = Vector(SIMD::g_trueMask); + + Vector vPi = Vector(SIMD::g_aTan2Constants).GetSplatX(); + Vector vPiOverTwo = Vector(SIMD::g_aTan2Constants).GetSplatY(); + Vector vPiOverFour = Vector(SIMD::g_aTan2Constants).GetSplatZ(); + Vector vThreePiOverFour = Vector(SIMD::g_aTan2Constants).GetSplatW(); + + Vector YEqualsZero = Y.EqualsZero(); + Vector XEqualsZero = X.EqualsZero(); + Vector XIsPositive = _mm_and_ps(X, SIMD::g_signMask); + XIsPositive = _mm_castsi128_ps(_mm_cmpeq_epi32(_mm_castps_si128(XIsPositive), _mm_castps_si128(Vector::Zero))); + Vector YEqualsInfinity = Y.EqualsInfinity(); + Vector XEqualsInfinity = X.EqualsInfinity(); + + Vector YSign = _mm_and_ps(Y, SIMD::g_signMask); + vPi = _mm_castsi128_ps(_mm_or_si128(_mm_castps_si128(vPi), _mm_castps_si128(YSign))); + vPiOverTwo = _mm_castsi128_ps(_mm_or_si128(_mm_castps_si128(vPiOverTwo), _mm_castps_si128(YSign))); + vPiOverFour = _mm_castsi128_ps(_mm_or_si128(_mm_castps_si128(vPiOverFour), _mm_castps_si128(YSign))); + vThreePiOverFour = _mm_castsi128_ps(_mm_or_si128(_mm_castps_si128(vThreePiOverFour), _mm_castps_si128(YSign))); + + Vector R1 = Vector::Select(vPi, YSign, XIsPositive); + Vector R2 = Vector::Select(ATanResultValid, vPiOverTwo, XEqualsZero); + Vector R3 = Vector::Select(R2, R1, YEqualsZero); + Vector R4 = Vector::Select(vThreePiOverFour, vPiOverFour, XIsPositive); + Vector R5 = Vector::Select(vPiOverTwo, R4, XEqualsInfinity); + Vector Result = Vector::Select(R3, R5, YEqualsInfinity); + ATanResultValid = _mm_castsi128_ps(_mm_cmpeq_epi32(_mm_castps_si128(Result), _mm_castps_si128(ATanResultValid))); + + Vector V = Y / X; + Vector R0 = Vector::ATan(V); + R1 = Vector::Select(vPi, Vector(SIMD::g_signMask), XIsPositive); + R2 = R0 + R1; + + return Vector::Select(Result, R2, ATanResultValid); + } + + FORCE_INLINE Vector Vector::SinEst(const Vector& vec) + { + // Force the value within the bounds of pi + auto m_x = Vector::AngleMod2Pi(vec); + + // Map in [-pi/2,pi/2] with sin(m_y) = sin(m_x). + __m128 sign = _mm_and_ps(m_x, SIMD::g_signMask); + __m128 c = _mm_or_ps(Vector::Pi, sign); // pi when m_x >= 0, -pi when m_x < 0 + __m128 absx = _mm_andnot_ps(sign, m_x); // |m_x| + __m128 rflx = _mm_sub_ps(c, m_x); + __m128 comp = _mm_cmple_ps(absx, Vector::PiDivTwo); + __m128 select0 = _mm_and_ps(comp, m_x); + __m128 select1 = _mm_andnot_ps(comp, rflx); + m_x = _mm_or_ps(select0, select1); + + __m128 x2 = _mm_mul_ps(m_x, m_x); + + // Compute polynomial approximation + const auto SEC = SIMD::g_sinCoefficients1; + auto vConstants = _mm_shuffle_ps(SEC, SEC, _MM_SHUFFLE(3, 3, 3, 3)); + __m128 Result = _mm_mul_ps(vConstants, x2); + + vConstants = _mm_shuffle_ps(SEC, SEC, _MM_SHUFFLE(2, 2, 2, 2)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(SEC, SEC, _MM_SHUFFLE(1, 1, 1, 1)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + Result = _mm_add_ps(Result, Vector::One); + Result = _mm_mul_ps(Result, m_x); + return Result; + } + + FORCE_INLINE Vector Vector::CosEst(const Vector& vec) + { + // Map V to m_x in [-pi,pi]. + auto m_x = Vector::AngleMod2Pi(vec); + + // Map in [-pi/2,pi/2] with cos(m_y) = sign*cos(m_x). + auto sign = _mm_and_ps(m_x, SIMD::g_signMask); + __m128 c = _mm_or_ps(Vector::Pi, sign); // pi when m_x >= 0, -pi when m_x < 0 + __m128 absx = _mm_andnot_ps(sign, m_x); // |m_x| + __m128 rflx = _mm_sub_ps(c, m_x); + __m128 comp = _mm_cmple_ps(absx, Vector::PiDivTwo); + __m128 select0 = _mm_and_ps(comp, m_x); + __m128 select1 = _mm_andnot_ps(comp, rflx); + m_x = _mm_or_ps(select0, select1); + select0 = _mm_and_ps(comp, Vector::One); + select1 = _mm_andnot_ps(comp, Vector::NegativeOne); + sign = _mm_or_ps(select0, select1); + + __m128 x2 = _mm_mul_ps(m_x, m_x); + + // Compute polynomial approximation + const auto CEC = SIMD::g_cosCoefficients1; + auto vConstants = _mm_shuffle_ps(CEC, CEC, _MM_SHUFFLE(3, 3, 3, 3)); + __m128 Result = _mm_mul_ps(vConstants, x2); + + vConstants = _mm_shuffle_ps(CEC, CEC, _MM_SHUFFLE(2, 2, 2, 2)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(CEC, CEC, _MM_SHUFFLE(1, 1, 1, 1)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + Result = _mm_add_ps(Result, Vector::One); + Result = _mm_mul_ps(Result, sign); + return Result; + } + + FORCE_INLINE Vector Vector::TanEst(const Vector& vec) + { + Vector W = Vector(SIMD::g_tanEstCoefficients).GetSplatW(); + Vector V1 = (vec * W).Round(); + V1 = Vector::NegativeMultiplySubtract(Vector::Pi, V1, vec); + + Vector const T0 = Vector(SIMD::g_tanEstCoefficients).GetSplatX(); + Vector const T1 = Vector(SIMD::g_tanEstCoefficients).GetSplatY(); + Vector const T2 = Vector(SIMD::g_tanEstCoefficients).GetSplatZ(); + + auto V2T2 = Vector::NegativeMultiplySubtract(V1, V1, T2); + auto V2 = V1 * V1; + auto V1T0 = V1 * T0; + auto V1T1 = V1 * T1; + + auto N = Vector::MultiplyAdd(V2, V1T1, V1T0); + auto D = V2T2.GetInverseEst(); + return N * D; + } + + FORCE_INLINE Vector Vector::ASinEst(const Vector& vec) + { + __m128 nonnegative = _mm_cmpge_ps(vec, Vector::Zero); + __m128 mvalue = _mm_sub_ps(Vector::Zero, vec); + __m128 m_x = _mm_max_ps(vec, mvalue); // |vec| + + // Compute (1-|vec|), clamp to zero to avoid sqrt of negative number. + __m128 oneMValue = _mm_sub_ps(Vector::One, m_x); + __m128 clampOneMValue = _mm_max_ps(Vector::Zero, oneMValue); + __m128 root = _mm_sqrt_ps(clampOneMValue); // sqrt(1-|vec|) + + // Compute polynomial approximation + const auto AEC = SIMD::g_arcEstCoefficients; + auto vConstants = _mm_shuffle_ps(AEC, AEC, _MM_SHUFFLE(3, 3, 3, 3)); + __m128 t0 = _mm_mul_ps(vConstants, m_x); + + vConstants = _mm_shuffle_ps(AEC, AEC, _MM_SHUFFLE(2, 2, 2, 2)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AEC, AEC, _MM_SHUFFLE(1, 1, 1, 1)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(AEC, AEC, _MM_SHUFFLE(0, 0, 0, 0)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, root); + + __m128 t1 = _mm_sub_ps(Vector::Pi, t0); + t0 = _mm_and_ps(nonnegative, t0); + t1 = _mm_andnot_ps(nonnegative, t1); + t0 = _mm_or_ps(t0, t1); + t0 = _mm_sub_ps(Vector::PiDivTwo, t0); + return t0; + } + + FORCE_INLINE Vector Vector::ACosEst(const Vector& vec) + { + __m128 nonnegative = _mm_cmpge_ps(vec, Vector::Zero); + __m128 mvalue = _mm_sub_ps(Vector::Zero, vec); + __m128 m_x = _mm_max_ps(vec, mvalue); // |vec| + + // Compute (1-|vec|), clamp to zero to avoid sqrt of negative number. + __m128 oneMValue = _mm_sub_ps(Vector::One, m_x); + __m128 clampOneMValue = _mm_max_ps(Vector::Zero, oneMValue); + __m128 root = _mm_sqrt_ps(clampOneMValue); // sqrt(1-|vec|) + + // Compute polynomial approximation + auto vConstants = _mm_shuffle_ps(SIMD::g_arcEstCoefficients, SIMD::g_arcEstCoefficients, _MM_SHUFFLE(3, 3, 3, 3)); + __m128 t0 = _mm_mul_ps(vConstants, m_x); + + vConstants = _mm_shuffle_ps(SIMD::g_arcEstCoefficients, SIMD::g_arcEstCoefficients, _MM_SHUFFLE(2, 2, 2, 2)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(SIMD::g_arcEstCoefficients, SIMD::g_arcEstCoefficients, _MM_SHUFFLE(1, 1, 1, 1)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, m_x); + + vConstants = _mm_shuffle_ps(SIMD::g_arcEstCoefficients, SIMD::g_arcEstCoefficients, _MM_SHUFFLE(0, 0, 0, 0)); + t0 = _mm_add_ps(t0, vConstants); + t0 = _mm_mul_ps(t0, root); + + __m128 t1 = _mm_sub_ps(Vector::Pi, t0); + t0 = _mm_and_ps(nonnegative, t0); + t1 = _mm_andnot_ps(nonnegative, t1); + t0 = _mm_or_ps(t0, t1); + return t0; + } + + FORCE_INLINE Vector Vector::ATanEst(const Vector& vec) + { + __m128 absV = vec.GetAbs(); + __m128 invV = _mm_div_ps(Vector::One, vec); + __m128 comp = _mm_cmpgt_ps(vec, Vector::One); + __m128 select0 = _mm_and_ps(comp, Vector::One); + __m128 select1 = _mm_andnot_ps(comp, Vector::NegativeOne); + __m128 sign = _mm_or_ps(select0, select1); + comp = _mm_cmple_ps(absV, Vector::One); + select0 = _mm_and_ps(comp, Vector::Zero); + select1 = _mm_andnot_ps(comp, sign); + sign = _mm_or_ps(select0, select1); + select0 = _mm_and_ps(comp, vec); + select1 = _mm_andnot_ps(comp, invV); + __m128 m_x = _mm_or_ps(select0, select1); + + __m128 x2 = _mm_mul_ps(m_x, m_x); + + // Compute polynomial approximation + Vector const AEC = SIMD::g_aTanEstCoefficients1; + Vector vConstants = _mm_shuffle_ps(AEC, AEC, _MM_SHUFFLE(3, 3, 3, 3)); + __m128 Result = _mm_mul_ps(vConstants, x2); + + vConstants = _mm_shuffle_ps(AEC, AEC, _MM_SHUFFLE(2, 2, 2, 2)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(AEC, AEC, _MM_SHUFFLE(1, 1, 1, 1)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(AEC, AEC, _MM_SHUFFLE(0, 0, 0, 0)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + // ATanEstCoefficients0 is already splatted + Result = _mm_add_ps(Result, SIMD::g_aTanEstCoefficients0); + Result = _mm_mul_ps(Result, m_x); + __m128 result1 = _mm_mul_ps(sign, Vector::PiDivTwo); + result1 = _mm_sub_ps(result1, Result); + + comp = _mm_cmpeq_ps(sign, Vector::Zero); + select0 = _mm_and_ps(comp, Result); + select1 = _mm_andnot_ps(comp, result1); + Result = _mm_or_ps(select0, select1); + return Result; + } + + FORCE_INLINE Vector Vector::ATan2Est(const Vector& X, const Vector& Y) + { + Vector ATanResultValid = Vector(SIMD::g_trueMask); + + Vector vPi = Vector(SIMD::g_aTan2Constants).GetSplatX(); + Vector vPiOverTwo = Vector(SIMD::g_aTan2Constants).GetSplatY(); + Vector vPiOverFour = Vector(SIMD::g_aTan2Constants).GetSplatZ(); + Vector vThreePiOverFour = Vector(SIMD::g_aTan2Constants).GetSplatW(); + + Vector YEqualsZero = Y.EqualsZero(); + Vector XEqualsZero = X.EqualsZero(); + Vector XIsPositive = _mm_and_ps(X, SIMD::g_signMask); + XIsPositive = _mm_castsi128_ps(_mm_cmpeq_epi32(_mm_castps_si128(XIsPositive), _mm_castps_si128(Vector::Zero))); + Vector YEqualsInfinity = Y.EqualsInfinity(); + Vector XEqualsInfinity = X.EqualsInfinity(); + + Vector YSign = _mm_and_ps(Y, SIMD::g_signMask); + vPi = _mm_castsi128_ps(_mm_or_si128(_mm_castps_si128(vPi), _mm_castps_si128(YSign))); + vPiOverTwo = _mm_castsi128_ps(_mm_or_si128(_mm_castps_si128(vPiOverTwo), _mm_castps_si128(YSign))); + vPiOverFour = _mm_castsi128_ps(_mm_or_si128(_mm_castps_si128(vPiOverFour), _mm_castps_si128(YSign))); + vThreePiOverFour = _mm_castsi128_ps(_mm_or_si128(_mm_castps_si128(vThreePiOverFour), _mm_castps_si128(YSign))); + + Vector R1 = Vector::Select(vPi, YSign, XIsPositive); + Vector R2 = Vector::Select(ATanResultValid, vPiOverTwo, XEqualsZero); + Vector R3 = Vector::Select(R2, R1, YEqualsZero); + Vector R4 = Vector::Select(vThreePiOverFour, vPiOverFour, XIsPositive); + Vector R5 = Vector::Select(vPiOverTwo, R4, XEqualsInfinity); + Vector Result = Vector::Select(R3, R5, YEqualsInfinity); + ATanResultValid = _mm_castsi128_ps(_mm_cmpeq_epi32(_mm_castps_si128(Result), _mm_castps_si128(ATanResultValid))); + + Vector Reciprocal = X.GetInverseEst(); + Vector V = Y * Reciprocal; + Vector R0 = Vector::ATanEst(V); + + R1 = Vector::Select(vPi, Vector(SIMD::g_signMask), XIsPositive); + R2 = R0 + R1; + Result = Vector::Select(Result, R2, ATanResultValid); + + return Result; + } + + FORCE_INLINE void Vector::SinCos(Vector& sin, Vector& cos, float angle) + { + return SinCos(sin, cos, Vector(angle)); + } + + FORCE_INLINE void Vector::SinCos(Vector& sin, Vector& cos, const Vector& angle) + { + // Force the value within the bounds of pi + auto m_x = Vector::AngleMod2Pi(angle); + + // Map in [-pi/2,pi/2] with sin(m_y) = sin(m_x), cos(m_y) = sign*cos(m_x). + auto sign = _mm_and_ps(m_x, SIMD::g_signMask); + __m128 c = _mm_or_ps(Vector::Pi, sign); // pi when m_x >= 0, -pi when m_x < 0 + __m128 absx = _mm_andnot_ps(sign, m_x); // |m_x| + __m128 rflx = _mm_sub_ps(c, m_x); + __m128 comp = _mm_cmple_ps(absx, Vector::PiDivTwo); + __m128 select0 = _mm_and_ps(comp, m_x); + __m128 select1 = _mm_andnot_ps(comp, rflx); + m_x = _mm_or_ps(select0, select1); + select0 = _mm_and_ps(comp, Vector::One); + select1 = _mm_andnot_ps(comp, Vector::NegativeOne); + sign = _mm_or_ps(select0, select1); + + __m128 x2 = _mm_mul_ps(m_x, m_x); + + // Compute polynomial approximation of sine + const auto SC1 = SIMD::g_sinCoefficients1; + auto vConstants = _mm_shuffle_ps(SC1, SC1, _MM_SHUFFLE(0, 0, 0, 0)); + __m128 Result = _mm_mul_ps(vConstants, x2); + + const auto SC0 = SIMD::g_sinCoefficients0; + vConstants = _mm_shuffle_ps(SC0, SC0, _MM_SHUFFLE(3, 3, 3, 3)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(SC0, SC0, _MM_SHUFFLE(2, 2, 2, 2)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(SC0, SC0, _MM_SHUFFLE(1, 1, 1, 1)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(SC0, SC0, _MM_SHUFFLE(0, 0, 0, 0)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + Result = _mm_add_ps(Result, Vector::One); + Result = _mm_mul_ps(Result, m_x); + sin = Result; + + // Compute polynomial approximation of cosine + const auto CC1 = SIMD::g_cosCoefficients1; + vConstants = _mm_shuffle_ps(CC1, CC1, _MM_SHUFFLE(0, 0, 0, 0)); + Result = _mm_mul_ps(vConstants, x2); + + const auto CC0 = SIMD::g_cosCoefficients0; + vConstants = _mm_shuffle_ps(CC0, CC0, _MM_SHUFFLE(3, 3, 3, 3)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(CC0, CC0, _MM_SHUFFLE(2, 2, 2, 2)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(CC0, CC0, _MM_SHUFFLE(1, 1, 1, 1)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + + vConstants = _mm_shuffle_ps(CC0, CC0, _MM_SHUFFLE(0, 0, 0, 0)); + Result = _mm_add_ps(Result, vConstants); + Result = _mm_mul_ps(Result, x2); + Result = _mm_add_ps(Result, Vector::One); + Result = _mm_mul_ps(Result, sign); + cos = Result; + } + + FORCE_INLINE Vector Vector::AngleMod2Pi(const Vector& angles) + { + // Modulo the range of the given angles such that -Pi <= Angles < Pi + Vector result = _mm_mul_ps(angles, Vector::OneDivTwoPi); + result.Round(); + result = _mm_mul_ps(result, Vector::TwoPi); + result = _mm_sub_ps(angles, result); + return result; + } + + FORCE_INLINE Vector::operator __m128& () + { + return m_data; + } + + FORCE_INLINE Vector::operator const __m128& () const + { + return m_data; + } + + FORCE_INLINE Vector::Vector() + { + } + + FORCE_INLINE Vector::Vector(Axis axis) + { + switch (axis) + { + case Axis::X: *this = Vector::UnitX; break; + case Axis::Y: *this = Vector::UnitY; break; + case Axis::Z: *this = Vector::UnitZ; break; + default: HALT(); break; + } + } + + FORCE_INLINE Vector::Vector(ZeroInit_t) + { + memset(this, 0, sizeof(Vector)); + } + + FORCE_INLINE Vector::Vector(float v) + { + m_data = _mm_set1_ps(v); + } + + FORCE_INLINE Vector::Vector(__m128 v) + : m_data(v) + { + } + + FORCE_INLINE Vector::Vector(float ix, float iy, float iz, float iw) + { + m_data = _mm_set_ps(iw, iz, iy, ix); + } + + FORCE_INLINE Vector::Vector(const Float2& v, float iz, float iw) + { + m_data = _mm_set_ps(iw, iz, v.m_y, v.m_x); + } + + FORCE_INLINE Vector::Vector(const Float3& v, float iw) + { + m_data = _mm_set_ps(iw, v.m_z, v.m_y, v.m_x); + } + + FORCE_INLINE Vector::Vector(const Float4& v) + { + m_data = _mm_loadu_ps(&v.m_x); + } + + FORCE_INLINE Vector::Vector(const float* pValues) + { + m_data = _mm_loadu_ps(pValues); + } + + FORCE_INLINE bool Vector::IsValid() const + { + return !IsNaN4() && !IsInfinite4(); + } + + FORCE_INLINE void Vector::Store(float* pValues) const + { + _mm_storeu_ps(pValues, m_data); + } + + FORCE_INLINE void Vector::StoreFloat(float& value) const + { + _mm_store_ss(&value, m_data); + } + + FORCE_INLINE void Vector::StoreFloat2(Float2& value) const + { + auto yVec = _mm_shuffle_ps(m_data, m_data, _MM_SHUFFLE(1, 1, 1, 1)); + _mm_store_ss(&value.m_x, m_data); + _mm_store_ss(&value.m_y, yVec); + } + + FORCE_INLINE void Vector::StoreFloat3(Float3& value) const + { + auto yVec = _mm_shuffle_ps(m_data, m_data, _MM_SHUFFLE(1, 1, 1, 1)); + auto zVec = _mm_shuffle_ps(m_data, m_data, _MM_SHUFFLE(2, 2, 2, 2)); + _mm_store_ss(&value.m_x, m_data); + _mm_store_ss(&value.m_y, yVec); + _mm_store_ss(&value.m_z, zVec); + } + + FORCE_INLINE void Vector::StoreFloat4(Float4& value) const + { + _mm_storeu_ps(&value.m_x, m_data); + } + + FORCE_INLINE float Vector::ToFloat() const + { + float v; + StoreFloat(v); + return v; + } + + FORCE_INLINE Float2 Vector::ToFloat2() const + { + Float2 v; + StoreFloat2(v); + return v; + } + + FORCE_INLINE Float3 Vector::ToFloat3() const + { + Float3 v; + StoreFloat3(v); + return v; + } + + FORCE_INLINE Float4 Vector::ToFloat4() const + { + Float4 v; + StoreFloat4(v); + return v; + } + + FORCE_INLINE Vector::operator Float2() const + { + return ToFloat2(); + } + + FORCE_INLINE Vector::operator Float3() const + { + return ToFloat3(); + } + + FORCE_INLINE Vector::operator Float4() const + { + return ToFloat4(); + } + + FORCE_INLINE float Vector::GetX() const + { + return _mm_cvtss_f32(m_data); + } + + FORCE_INLINE float Vector::GetY() const + { + auto vTemp = GetSplatY(); + return _mm_cvtss_f32(vTemp); + } + + FORCE_INLINE float Vector::GetZ() const + { + auto vTemp = GetSplatZ(); + return _mm_cvtss_f32(vTemp); + } + + FORCE_INLINE float Vector::GetW() const + { + auto vTemp = GetSplatW(); + return _mm_cvtss_f32(vTemp); + } + + FORCE_INLINE void Vector::SetX(float x) + { + m_data = _mm_move_ss(m_data, _mm_set_ss(x)); + } + + FORCE_INLINE void Vector::SetY(float y) + { + m_data = _mm_insert_ps(m_data, _mm_set_ss(y), 0x10); + } + + FORCE_INLINE void Vector::SetZ(float z) + { + m_data = _mm_insert_ps(m_data, _mm_set_ss(z), 0x20); + } + + FORCE_INLINE void Vector::SetW(float w) + { + m_data = _mm_insert_ps(m_data, _mm_set_ss(w), 0x30); + } + + FORCE_INLINE float Vector::operator[](uint32_t i) const + { + ASSERT(i < 4); + + switch (i) + { + case 0: return GetX(); break; + case 1: return GetY(); break; + case 2: return GetZ(); break; + case 3: return GetW(); break; + } + + UNREACHABLE_CODE(); + return 0.0f; + } + + FORCE_INLINE bool Vector::IsW1() const + { + return GetSplatW().IsEqual4(Vector::One); + } + + FORCE_INLINE bool Vector::IsW0() const + { + return GetSplatW().IsZero4(); + } + + FORCE_INLINE Vector& Vector::SetW0() + { + SetW(0.0f); + return *this; + } + + FORCE_INLINE Vector& Vector::SetW1() + { + SetW(1.0f); + return *this; + } + + FORCE_INLINE Vector Vector::GetWithW0() const + { + Vector v = *this; + v.SetW0(); + return v; + } + + FORCE_INLINE Vector Vector::GetWithW1() const + { + Vector v = *this; + v.SetW1(); + return v; + } + + FORCE_INLINE Vector Vector::Get2D() const + { + return Vector::Select(*this, Vector::Zero, Vector::Select0011); + } + + FORCE_INLINE Vector Vector::Get3D() const + { + return Vector::Select(*this, Vector::Zero, Vector::Select0001); + } + + FORCE_INLINE Vector Vector::operator+(const Vector& v) const + { + return _mm_add_ps(m_data, v); + } + + FORCE_INLINE Vector& Vector::operator+=(const Vector& v) + { + m_data = _mm_add_ps(m_data, v); + return *this; + } + + FORCE_INLINE Vector Vector::operator-(const Vector& v) const + { + return _mm_sub_ps(m_data, v); + } + + FORCE_INLINE Vector& Vector::operator-=(const Vector& v) + { + m_data = _mm_sub_ps(m_data, v); + return *this; + } + + FORCE_INLINE Vector Vector::operator*(const Vector& v) const + { + return _mm_mul_ps(m_data, v); + } + + FORCE_INLINE Vector& Vector::operator*=(const Vector& v) + { + m_data = _mm_mul_ps(m_data, v); + return *this; + } + + FORCE_INLINE Vector Vector::operator/(const Vector& v) const + { + return _mm_div_ps(m_data, v); + } + + FORCE_INLINE Vector& Vector::operator/=(const Vector& v) + { + m_data = _mm_div_ps(m_data, v); + return *this; + } + + FORCE_INLINE Vector Vector::operator*(float const f) const + { + return operator*(Vector(f)); + } + + FORCE_INLINE Vector& Vector::operator*=(float const f) + { + return operator*=(Vector(f)); + } + + FORCE_INLINE Vector Vector::operator/(float const f) const + { + return operator/(Vector(f)); + } + + FORCE_INLINE Vector& Vector::operator/=(float const f) + { + return operator/=(Vector(f)); + } + + FORCE_INLINE Vector Vector::operator-() const + { + return GetNegated(); + } + + FORCE_INLINE Vector Vector::Orthogonal2D() const + { + static Vector const negX(-1.0f, 1.0f, 1.0f, 1.0f); + + Vector result; + result = _mm_shuffle_ps(*this, *this, _MM_SHUFFLE(3, 2, 0, 1)); + result = _mm_mul_ps(result, negX); + return result; + } + + FORCE_INLINE Vector Vector::Cross2(const Vector& other) const + { + Vector vResult = _mm_shuffle_ps(other.m_data, other.m_data, _MM_SHUFFLE(0, 1, 0, 1)); + vResult = _mm_mul_ps(vResult, m_data); + Vector vTemp = vResult.GetSplatY(); + vResult = _mm_sub_ss(vResult, vTemp); + vResult = vResult.GetSplatX(); + return vResult; + } + + FORCE_INLINE Vector Vector::Cross3(const Vector& other) const + { + auto vTemp1 = _mm_shuffle_ps(m_data, m_data, _MM_SHUFFLE(3, 0, 2, 1)); + auto vTemp2 = _mm_shuffle_ps(other, other, _MM_SHUFFLE(3, 1, 0, 2)); + Vector result = _mm_mul_ps(vTemp1, vTemp2); + vTemp1 = _mm_shuffle_ps(vTemp1, vTemp1, _MM_SHUFFLE(3, 0, 2, 1)); + vTemp2 = _mm_shuffle_ps(vTemp2, vTemp2, _MM_SHUFFLE(3, 1, 0, 2)); + vTemp1 = _mm_mul_ps(vTemp1, vTemp2); + result = _mm_sub_ps(result, vTemp1); + result = _mm_and_ps(result, SIMD::g_maskXYZ0); + return result; + } + + FORCE_INLINE Vector Vector::Dot2(const Vector& other) const + { + // Perform the dot product on m_x and m_y + Vector result = _mm_mul_ps(m_data, other); + // vTemp has m_y splatted + auto vTemp = _mm_shuffle_ps(result, result, _MM_SHUFFLE(1, 1, 1, 1)); + // m_x+m_y + result = _mm_add_ss(result, vTemp); + result = _mm_shuffle_ps(result, result, _MM_SHUFFLE(0, 0, 0, 0)); + return result; + } + + FORCE_INLINE Vector Vector::Dot3(const Vector& vOther) const + { + // Perform the dot product + auto vDot = _mm_mul_ps(m_data, vOther); + // m_x=Dot.vector4_f32[1], m_y=Dot.vector4_f32[2] + auto vTemp = _mm_shuffle_ps(vDot, vDot, _MM_SHUFFLE(2, 1, 2, 1)); + // Result.vector4_f32[0] = m_x+m_y + vDot = _mm_add_ss(vDot, vTemp); + // m_x=Dot.vector4_f32[2] + vTemp = _mm_shuffle_ps(vTemp, vTemp, _MM_SHUFFLE(1, 1, 1, 1)); + // Result.vector4_f32[0] = (m_x+m_y)+m_z + vDot = _mm_add_ss(vDot, vTemp); + // Splat m_x + Vector result = _mm_shuffle_ps(vDot, vDot, _MM_SHUFFLE(0, 0, 0, 0)); + return result; + } + + FORCE_INLINE Vector Vector::Dot4(const Vector& other) const + { + auto vTemp2 = other; + auto vTemp = _mm_mul_ps(m_data, vTemp2); + vTemp2 = _mm_shuffle_ps(vTemp2, vTemp, _MM_SHUFFLE(1, 0, 0, 0)); // Copy X to the Z position and Y to the W position + vTemp2 = _mm_add_ps(vTemp2, vTemp); // Add Z = X+Z; W = Y+W; + vTemp = _mm_shuffle_ps(vTemp, vTemp2, _MM_SHUFFLE(0, 3, 0, 0)); // Copy W to the Z position + vTemp = _mm_add_ps(vTemp, vTemp2); // Add Z and W together + return _mm_shuffle_ps(vTemp, vTemp, _MM_SHUFFLE(2, 2, 2, 2)); // Splat Z and return + } + + FORCE_INLINE float Vector::GetDot2(const Vector& other) const + { + return Dot2(other).ToFloat(); + } + + FORCE_INLINE float Vector::GetDot3(const Vector& other) const + { + return Dot3(other).ToFloat(); + } + + FORCE_INLINE float Vector::GetDot4(const Vector& other) const + { + return Dot4(other).ToFloat(); + } + + FORCE_INLINE Vector Vector::ScalarProjection(const Vector& other) const + { + Vector const normalizedThis = GetNormalized3(); + Vector const projection = other.Dot3(normalizedThis); + return projection; + } + + FORCE_INLINE float Vector::GetScalarProjection(const Vector& other) const + { + return ScalarProjection(other).ToFloat(); + } + + FORCE_INLINE Vector Vector::VectorProjection(const Vector& other) const + { + Vector const normalizedThis = GetNormalized3(); + Vector const dotOther = other.Dot3(normalizedThis); + Vector const projection = normalizedThis * dotOther; + return projection; + } + + FORCE_INLINE Vector& Vector::Invert() + { + m_data = _mm_div_ps(Vector::One, m_data); + return *this; + } + + FORCE_INLINE Vector Vector::GetInverse() const + { + return _mm_div_ps(Vector::One, m_data); + } + + FORCE_INLINE Vector Vector::GetReciprocal() const + { + return GetInverse(); + } + + FORCE_INLINE Vector& Vector::InvertEst() + { + m_data = _mm_rcp_ps(m_data); + return *this; + } + + FORCE_INLINE Vector Vector::GetInverseEst() const + { + return _mm_rcp_ps(m_data); + } + + FORCE_INLINE Vector& Vector::Negate() + { + m_data = _mm_sub_ps(Vector::Zero, m_data); + return *this; + } + + FORCE_INLINE Vector Vector::GetNegated() const + { + return _mm_sub_ps(Vector::Zero, m_data); + } + + FORCE_INLINE Vector& Vector::Abs() + { + m_data = _mm_max_ps(_mm_sub_ps(Vector::Zero, m_data), m_data); + return *this; + } + + FORCE_INLINE Vector Vector::GetAbs() const + { + return _mm_max_ps(_mm_sub_ps(Vector::Zero, m_data), m_data); + } + + FORCE_INLINE Vector& Vector::Sqrt() + { + m_data = _mm_sqrt_ps(m_data); + return *this; + } + + FORCE_INLINE Vector Vector::GetSqrt() + { + return _mm_sqrt_ps(m_data); + } + + FORCE_INLINE Vector& Vector::ReciprocalSqrt() + { + m_data = _mm_div_ps(Vector::One, _mm_sqrt_ps(m_data)); + return *this; + } + + FORCE_INLINE Vector Vector::GetReciprocalSqrt() + { + return _mm_div_ps(Vector::One, _mm_sqrt_ps(m_data)); + } + + FORCE_INLINE Vector& Vector::EstimatedReciprocalSqrt() + { + m_data = _mm_rsqrt_ps(m_data); + return *this; + } + + FORCE_INLINE Vector Vector::GetEstimatedReciprocalSqrt() + { + return _mm_rsqrt_ps(m_data); + } + + FORCE_INLINE Vector& Vector::Normalize2() + { + // Perform the dot product on m_x and m_y only + auto vLengthSq = _mm_mul_ps(m_data, m_data); + auto vTemp = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(1, 1, 1, 1)); + vLengthSq = _mm_add_ss(vLengthSq, vTemp); + vLengthSq = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(0, 0, 0, 0)); + // Prepare for the division + auto vResult = _mm_sqrt_ps(vLengthSq); + // Create zero with a single instruction + auto vZeroMask = _mm_setzero_ps(); + // Test for a divide by zero (Must be FP to detect -0.0) + vZeroMask = _mm_cmpneq_ps(vZeroMask, vResult); + // Failsafe on zero (Or epsilon) length planes + // If the length is infinity, set the elements to zero + vLengthSq = _mm_cmpneq_ps(vLengthSq, Vector::Infinity); + // Divide to perform the normalization + vResult = _mm_div_ps(m_data, vResult); + // Any that are infinity, set to zero + vResult = _mm_and_ps(vResult, vZeroMask); + // Select qnan or result based on infinite length + auto vTemp1 = _mm_andnot_ps(vLengthSq, Vector::QNaN); + auto vTemp2 = _mm_and_ps(vResult, vLengthSq); + m_data = _mm_or_ps(vTemp1, vTemp2); + + *this = Select(*this, Vector::Zero, Select0011); + + return *this; + } + + FORCE_INLINE Vector& Vector::Normalize3() + { + // Perform the dot product on m_x,m_y and m_z only + auto vLengthSq = _mm_mul_ps(m_data, m_data); + auto vTemp = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(2, 1, 2, 1)); + vLengthSq = _mm_add_ss(vLengthSq, vTemp); + vTemp = _mm_shuffle_ps(vTemp, vTemp, _MM_SHUFFLE(1, 1, 1, 1)); + vLengthSq = _mm_add_ss(vLengthSq, vTemp); + vLengthSq = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(0, 0, 0, 0)); + // Prepare for the division + auto vResult = _mm_sqrt_ps(vLengthSq); + // Create zero with a single instruction + auto vZeroMask = _mm_setzero_ps(); + // Test for a divide by zero (Must be FP to detect -0.0) + vZeroMask = _mm_cmpneq_ps(vZeroMask, vResult); + // Failsafe on zero (Or epsilon) length planes + // If the length is infinity, set the elements to zero + vLengthSq = _mm_cmpneq_ps(vLengthSq, Vector::Infinity); + // Divide to perform the normalization + vResult = _mm_div_ps(m_data, vResult); + // Any that are infinity, set to zero + vResult = _mm_and_ps(vResult, vZeroMask); + // Select qnan or result based on infinite length + auto vTemp1 = _mm_andnot_ps(vLengthSq, Vector::QNaN); + auto vTemp2 = _mm_and_ps(vResult, vLengthSq); + m_data = _mm_or_ps(vTemp1, vTemp2); + + *this = Select(*this, Vector::Zero, Select0001); + + return *this; + } + + FORCE_INLINE Vector& Vector::Normalize4() + { + // Perform the dot product on m_x,m_y,m_z and m_w + auto vLengthSq = _mm_mul_ps(m_data, m_data); + // vTemp has m_z and m_w + auto vTemp = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(3, 2, 3, 2)); + // m_x+m_z, m_y+m_w + vLengthSq = _mm_add_ps(vLengthSq, vTemp); + // m_x+m_z,m_x+m_z,m_x+m_z,m_y+m_w + vLengthSq = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(1, 0, 0, 0)); + // ??,??,m_y+m_w,m_y+m_w + vTemp = _mm_shuffle_ps(vTemp, vLengthSq, _MM_SHUFFLE(3, 3, 0, 0)); + // ??,??,m_x+m_z+m_y+m_w,?? + vLengthSq = _mm_add_ps(vLengthSq, vTemp); + // Splat the length + vLengthSq = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(2, 2, 2, 2)); + // Prepare for the division + auto vResult = _mm_sqrt_ps(vLengthSq); + // Create zero with a single instruction + auto vZeroMask = _mm_setzero_ps(); + // Test for a divide by zero (Must be FP to detect -0.0) + vZeroMask = _mm_cmpneq_ps(vZeroMask, vResult); + // Failsafe on zero (Or epsilon) length planes + // If the length is infinity, set the elements to zero + vLengthSq = _mm_cmpneq_ps(vLengthSq, Vector::Infinity); + // Divide to perform the normalization + vResult = _mm_div_ps(m_data, vResult); + // Any that are infinity, set to zero + vResult = _mm_and_ps(vResult, vZeroMask); + // Select qnan or result based on infinite length + auto vTemp1 = _mm_andnot_ps(vLengthSq, Vector::QNaN); + auto vTemp2 = _mm_and_ps(vResult, vLengthSq); + m_data = _mm_or_ps(vTemp1, vTemp2); + + return *this; + } + + FORCE_INLINE Vector Vector::GetNormalized2() const + { + Vector v = *this; + v.Normalize2(); + return v; + } + + FORCE_INLINE Vector Vector::GetNormalized3() const + { + Vector v = *this; + v.Normalize3(); + return v; + } + + FORCE_INLINE Vector Vector::GetNormalized4() const + { + Vector v = *this; + v.Normalize4(); + return v; + } + + FORCE_INLINE Vector& Vector::Floor() + { + Vector result; + + // To handle NAN, INF and numbers greater than 8388608, use masking + __m128i vTest = _mm_and_si128(_mm_castps_si128(m_data), SIMD::g_absMask); + vTest = _mm_cmplt_epi32(vTest, SIMD::g_noFraction); + // Truncate + __m128i vInt = _mm_cvttps_epi32(m_data); + result = _mm_cvtepi32_ps(vInt); + __m128 vLarger = _mm_cmpgt_ps(result, m_data); + // 0 -> 0, 0xffffffff -> -1.0f + vLarger = _mm_cvtepi32_ps(_mm_castps_si128(vLarger)); + result = _mm_add_ps(result, vLarger); + // All numbers less than 8388608 will use the round to int + result = _mm_and_ps(result, _mm_castsi128_ps(vTest)); + // All others, use the ORIGINAL value + vTest = _mm_andnot_si128(vTest, _mm_castps_si128(m_data)); + result = _mm_or_ps(result, _mm_castsi128_ps(vTest)); + + m_data = result; + return *this; + } + + FORCE_INLINE Vector Vector::GetFloor() const + { + Vector v = *this; + v.Floor(); + return v; + } + + FORCE_INLINE Vector& Vector::Ceil() + { + Vector result; + + // To handle NAN, INF and numbers greater than 8388608, use masking + __m128i vTest = _mm_and_si128(_mm_castps_si128(m_data), SIMD::g_absMask); + vTest = _mm_cmplt_epi32(vTest, SIMD::g_noFraction); + // Truncate + __m128i vInt = _mm_cvttps_epi32(m_data); + result = _mm_cvtepi32_ps(vInt); + __m128 vSmaller = _mm_cmplt_ps(result, m_data); + // 0 -> 0, 0xffffffff -> -1.0f + vSmaller = _mm_cvtepi32_ps(_mm_castps_si128(vSmaller)); + result = _mm_sub_ps(result, vSmaller); + // All numbers less than 8388608 will use the round to int + result = _mm_and_ps(result, _mm_castsi128_ps(vTest)); + // All others, use the ORIGINAL value + vTest = _mm_andnot_si128(vTest, _mm_castps_si128(m_data)); + result = _mm_or_ps(result, _mm_castsi128_ps(vTest)); + + m_data = result; + return *this; + } + + FORCE_INLINE Vector Vector::GetCeil() const + { + Vector v = *this; + v.Ceil(); + return v; + } + + FORCE_INLINE Vector& Vector::Round() + { + __m128 sign = _mm_and_ps(m_data, SIMD::g_signMask); + __m128 sMagic = _mm_or_ps(SIMD::g_noFraction, sign); + __m128 R1 = _mm_add_ps(m_data, sMagic); + R1 = _mm_sub_ps(R1, sMagic); + __m128 R2 = _mm_and_ps(m_data, SIMD::g_absMask); + __m128 mask = _mm_cmple_ps(R2, SIMD::g_noFraction); + R2 = _mm_andnot_ps(mask, m_data); + R1 = _mm_and_ps(R1, mask); + m_data = _mm_xor_ps(R1, R2); + return *this; + } + + FORCE_INLINE Vector Vector::GetRound() const + { + Vector v = *this; + v.Round(); + return v; + } + + FORCE_INLINE Vector Vector::GetSign() const + { + Vector const selectMask = GreaterThanEqual(Vector::Zero); + return Vector::Select(Vector::NegativeOne, Vector::One, selectMask); + } + + FORCE_INLINE Vector Vector::GetSplatX() const + { + return _mm_shuffle_ps(m_data, m_data, _MM_SHUFFLE(0, 0, 0, 0)); + } + + FORCE_INLINE Vector Vector::GetSplatY() const + { + return _mm_shuffle_ps(m_data, m_data, _MM_SHUFFLE(1, 1, 1, 1)); + } + + FORCE_INLINE Vector Vector::GetSplatZ() const + { + return _mm_shuffle_ps(m_data, m_data, _MM_SHUFFLE(2, 2, 2, 2)); + } + + FORCE_INLINE Vector Vector::GetSplatW() const + { + return _mm_shuffle_ps(m_data, m_data, _MM_SHUFFLE(3, 3, 3, 3)); + } + + template + FORCE_INLINE Vector Vector::Swizzle() const + { + static_assert(xIdx < 4, "Element index parameter out of range"); + static_assert(yIdx < 4, "Element index parameter out of range"); + static_assert(zIdx < 4, "Element index parameter out of range"); + static_assert(wIdx < 4, "Element index parameter out of range"); + return _mm_shuffle_ps(m_data, m_data, _MM_SHUFFLE(wIdx, zIdx, yIdx, xIdx)); + } + + FORCE_INLINE Vector Vector::Swizzle(uint32_t xIdx, uint32_t yIdx, uint32_t zIdx, uint32_t wIdx) const + { + ASSERT(xIdx < 4 && yIdx < 4 && zIdx < 4 && wIdx < 4); + uint32_t const elem[4] = { xIdx, yIdx, zIdx, wIdx }; + __m128i vControl = _mm_loadu_si128(reinterpret_cast(&elem[0])); + return _mm_permutevar_ps(m_data, vControl); + } + + FORCE_INLINE Vector Vector::Shuffle(uint32_t xIdx, uint32_t yIdx, uint32_t zIdx, uint32_t wIdx) const + { + return Swizzle(xIdx, yIdx, zIdx, wIdx); + } + + template + FORCE_INLINE Vector Vector::Shuffle() const + { + return Swizzle(); + } + + FORCE_INLINE Vector Vector::Length2() const + { + Vector result; + + result = _mm_mul_ps(m_data, m_data); + auto vTemp = _mm_shuffle_ps(result, result, _MM_SHUFFLE(1, 1, 1, 1)); + // m_x+m_y + result = _mm_add_ss(result, vTemp); + result = _mm_shuffle_ps(result, result, _MM_SHUFFLE(0, 0, 0, 0)); + result = _mm_sqrt_ps(result); + return result; + } + + FORCE_INLINE Vector Vector::Length3() const + { + Vector result; + + // Perform the dot product on m_x,m_y and m_z + result = _mm_mul_ps(m_data, m_data); + // vTemp has m_z and m_y + auto vTemp = _mm_shuffle_ps(result, result, _MM_SHUFFLE(1, 2, 1, 2)); + // m_x+m_z, m_y + result = _mm_add_ss(result, vTemp); + // m_y,m_y,m_y,m_y + vTemp = _mm_shuffle_ps(vTemp, vTemp, _MM_SHUFFLE(1, 1, 1, 1)); + // m_x+m_z+m_y,??,??,?? + result = _mm_add_ss(result, vTemp); + // Splat the length squared + result = _mm_shuffle_ps(result, result, _MM_SHUFFLE(0, 0, 0, 0)); + // Get the length + result = _mm_sqrt_ps(result); + + return result; + } + + FORCE_INLINE Vector Vector::Length4() const + { + Vector result; + + // Perform the dot product on m_x,m_y,m_z and m_w + result = _mm_mul_ps(m_data, m_data); + // vTemp has m_z and m_w + auto vTemp = _mm_shuffle_ps(result, result, _MM_SHUFFLE(3, 2, 3, 2)); + // m_x+m_z, m_y+m_w + result = _mm_add_ps(result, vTemp); + // m_x+m_z,m_x+m_z,m_x+m_z,m_y+m_w + result = _mm_shuffle_ps(result, result, _MM_SHUFFLE(1, 0, 0, 0)); + // ??,??,m_y+m_w,m_y+m_w + vTemp = _mm_shuffle_ps(vTemp, result, _MM_SHUFFLE(3, 3, 0, 0)); + // ??,??,m_x+m_z+m_y+m_w,?? + result = _mm_add_ps(result, vTemp); + // Splat the length + result = _mm_shuffle_ps(result, result, _MM_SHUFFLE(2, 2, 2, 2)); + // Get the length + result = _mm_sqrt_ps(result); + + return result; + } + + FORCE_INLINE float Vector::GetLength2() const + { + return Length2().GetX(); + } + + FORCE_INLINE float Vector::GetLength3() const + { + return Length3().GetX(); + } + + FORCE_INLINE float Vector::GetLength4() const + { + return Length4().GetX(); + } + + FORCE_INLINE Vector Vector::InverseLength2() const + { + // Perform the dot product on m_x and m_y + auto vLengthSq = _mm_mul_ps(m_data, m_data); + // vTemp has m_y splatted + auto vTemp = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(1, 1, 1, 1)); + // m_x+m_y + vLengthSq = _mm_add_ss(vLengthSq, vTemp); + vLengthSq = _mm_sqrt_ss(vLengthSq); + vLengthSq = _mm_div_ss(Vector::One, vLengthSq); + vLengthSq = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(0, 0, 0, 0)); + return vLengthSq; + } + + FORCE_INLINE Vector Vector::InverseLength3() const + { + // Perform the dot product + auto vDot = _mm_mul_ps(m_data, m_data); + // m_x=Dot.m_y, m_y=Dot.m_z + auto vTemp = _mm_shuffle_ps(vDot, vDot, _MM_SHUFFLE(2, 1, 2, 1)); + // Result.m_x = m_x+m_y + vDot = _mm_add_ss(vDot, vTemp); + // m_x=Dot.m_z + vTemp = _mm_shuffle_ps(vTemp, vTemp, _MM_SHUFFLE(1, 1, 1, 1)); + // Result.m_x = (m_x+m_y)+m_z + vDot = _mm_add_ss(vDot, vTemp); + // Splat m_x + vDot = _mm_shuffle_ps(vDot, vDot, _MM_SHUFFLE(0, 0, 0, 0)); + // Get the reciprocal + vDot = _mm_sqrt_ps(vDot); + // Get the reciprocal + vDot = _mm_div_ps(Vector::One, vDot); + return vDot; + } + + FORCE_INLINE Vector Vector::InverseLength4() const + { + // Perform the dot product on m_x,m_y,m_z and m_w + auto vLengthSq = _mm_mul_ps(m_data, m_data); + // vTemp has m_z and m_w + auto vTemp = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(3, 2, 3, 2)); + // m_x+m_z, m_y+m_w + vLengthSq = _mm_add_ps(vLengthSq, vTemp); + // m_x+m_z,m_x+m_z,m_x+m_z,m_y+m_w + vLengthSq = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(1, 0, 0, 0)); + // ??,??,m_y+m_w,m_y+m_w + vTemp = _mm_shuffle_ps(vTemp, vLengthSq, _MM_SHUFFLE(3, 3, 0, 0)); + // ??,??,m_x+m_z+m_y+m_w,?? + vLengthSq = _mm_add_ps(vLengthSq, vTemp); + // Splat the length + vLengthSq = _mm_shuffle_ps(vLengthSq, vLengthSq, _MM_SHUFFLE(2, 2, 2, 2)); + // Get the reciprocal + vLengthSq = _mm_sqrt_ps(vLengthSq); + // Accurate! + vLengthSq = _mm_div_ps(Vector::One, vLengthSq); + return vLengthSq; + } + + FORCE_INLINE float Vector::GetInverseLength2() const + { + return InverseLength2().GetX(); + } + + FORCE_INLINE float Vector::GetInverseLength3() const + { + return InverseLength3().GetX(); + } + + FORCE_INLINE float Vector::GetInverseLength4() const + { + return InverseLength4().GetX(); + } + + FORCE_INLINE Vector Vector::LengthSquared2() const + { + return Vector::Dot2(m_data, m_data); + } + + FORCE_INLINE Vector Vector::LengthSquared3() const + { + return Vector::Dot3(m_data, m_data); + } + + FORCE_INLINE Vector Vector::LengthSquared4() const + { + return Vector::Dot4(m_data, m_data); + } + + FORCE_INLINE float Vector::GetLengthSquared2() const + { + return LengthSquared2().GetX(); + } + + FORCE_INLINE float Vector::GetLengthSquared3() const + { + return LengthSquared3().GetX(); + } + + FORCE_INLINE float Vector::GetLengthSquared4() const + { + return LengthSquared4().GetX(); + } + + FORCE_INLINE Vector Vector::Distance2(const Vector& to) const + { + return (to - *this).Length2(); + } + + FORCE_INLINE Vector Vector::Distance3(const Vector& to) const + { + return (to - *this).Length3(); + } + + FORCE_INLINE Vector Vector::Distance4(const Vector& to) const + { + return (to - *this).Length4(); + } + + FORCE_INLINE float Vector::GetDistance2(const Vector& to) const + { + return (to - *this).Length2().GetX(); + } + + FORCE_INLINE float Vector::GetDistance3(const Vector& to) const + { + return (to - *this).Length3().GetX(); + } + + FORCE_INLINE float Vector::GetDistance4(const Vector& to) const + { + return (to - *this).Length4().GetX(); + } + + FORCE_INLINE Vector Vector::DistanceSquared2(const Vector& to) const + { + return (to - *this).LengthSquared2(); + } + + FORCE_INLINE Vector Vector::DistanceSquared3(const Vector& to) const + { + return (to - *this).LengthSquared3(); + } + + FORCE_INLINE Vector Vector::DistanceSquared4(const Vector& to) const + { + return (to - *this).LengthSquared4(); + } + + FORCE_INLINE float Vector::GetDistanceSquared2(const Vector& to) const + { + return (to - *this).GetLengthSquared2(); + } + + FORCE_INLINE float Vector::GetDistanceSquared3(const Vector& to) const + { + return (to - *this).GetLengthSquared3(); + } + + FORCE_INLINE float Vector::GetDistanceSquared4(const Vector& to) const + { + return (to - *this).GetLengthSquared4(); + } + + FORCE_INLINE bool Vector::IsNormalized2() const + { + return (LengthSquared2() - Vector::One).Abs().IsLessThanEqual4(Vector::NormalizeCheckThreshold); + } + + FORCE_INLINE bool Vector::IsNormalized3() const + { + return (LengthSquared3() - Vector::One).Abs().IsLessThanEqual4(Vector::NormalizeCheckThreshold); + } + + FORCE_INLINE bool Vector::IsNormalized4() const + { + return (LengthSquared4() - Vector::One).Abs().IsLessThanEqual4(Vector::NormalizeCheckThreshold); + } + + FORCE_INLINE Vector Vector::InBounds(const Vector& bounds) const + { + // Test if less than or equal + auto vTemp1 = _mm_cmple_ps(m_data, bounds); + // Negate the bounds + auto vTemp2 = _mm_mul_ps(bounds, Vector::NegativeOne); + // Test if greater or equal (Reversed) + vTemp2 = _mm_cmple_ps(vTemp2, m_data); + // Blend answers + vTemp1 = _mm_and_ps(vTemp1, vTemp2); + return vTemp1; + } + + FORCE_INLINE bool Vector::IsInBounds2(const Vector& bounds) const + { + return ((_mm_movemask_ps(InBounds(bounds)) & 0x3) == 0x3) != 0; + } + + FORCE_INLINE bool Vector::IsInBounds3(const Vector& bounds) const + { + return ((_mm_movemask_ps(InBounds(bounds)) & 0x7) == 0x7) != 0; + } + + FORCE_INLINE bool Vector::IsInBounds4(const Vector& bounds) const + { + return (_mm_movemask_ps(InBounds(bounds)) == 0x0f) != 0; + } + + FORCE_INLINE Vector Vector::Equal(const Vector& v) const + { + return _mm_cmpeq_ps(*this, v); + } + + FORCE_INLINE bool Vector::IsEqual2(const Vector& v) const + { + return (((_mm_movemask_ps(Equal(v)) & 3) == 3) != 0); + } + + FORCE_INLINE bool Vector::IsEqual3(const Vector& v) const + { + return (((_mm_movemask_ps(Equal(v)) & 7) == 7) != 0); + } + + FORCE_INLINE bool Vector::IsEqual4(const Vector& v) const + { + return ((_mm_movemask_ps(Equal(v)) == 0x0f) != 0); + } + + FORCE_INLINE Vector Vector::NearEqual(const Vector& v, const Vector& epsilon) const + { + // Get the difference + auto vDelta = _mm_sub_ps(m_data, v); + // Get the absolute value of the difference + auto vTemp = _mm_setzero_ps(); + vTemp = _mm_sub_ps(vTemp, vDelta); + vTemp = _mm_max_ps(vTemp, vDelta); + vTemp = _mm_cmple_ps(vTemp, epsilon); + return vTemp; + } + + FORCE_INLINE bool Vector::IsNearEqual2(const Vector& v, float epsilon) const + { + return IsNearEqual2(v, Vector(epsilon)); + } + + FORCE_INLINE bool Vector::IsNearEqual3(const Vector& v, float epsilon) const + { + return IsNearEqual3(v, Vector(epsilon)); + } + + FORCE_INLINE bool Vector::IsNearEqual4(const Vector& v, float epsilon) const + { + return IsNearEqual4(v, Vector(epsilon)); + } + + FORCE_INLINE bool Vector::IsNearEqual2(const Vector& v, const Vector& epsilon) const + { + return (((_mm_movemask_ps(NearEqual(v, epsilon)) & 3) == 0x3) != 0); + } + + FORCE_INLINE bool Vector::IsNearEqual3(const Vector& v, const Vector& epsilon) const + { + return (((_mm_movemask_ps(NearEqual(v, epsilon)) & 7) == 0x7) != 0); + } + + FORCE_INLINE bool Vector::IsNearEqual4(const Vector& v, const Vector& epsilon) const + { + return ((_mm_movemask_ps(NearEqual(v, epsilon)) == 0xf) != 0); + } + + FORCE_INLINE Vector Vector::GreaterThan(const Vector& v) const + { + return _mm_cmpgt_ps(m_data, v); + } + + FORCE_INLINE bool Vector::IsAnyGreaterThan(const Vector& v) const + { + return !GreaterThan(v).IsZero4(); + } + + FORCE_INLINE bool Vector::IsGreaterThan2(const Vector& v) const + { + return (((_mm_movemask_ps(GreaterThan(v)) & 3) == 3) != 0); + } + + FORCE_INLINE bool Vector::IsGreaterThan3(const Vector& v) const + { + return (((_mm_movemask_ps(GreaterThan(v)) & 7) == 7) != 0); + } + + FORCE_INLINE bool Vector::IsGreaterThan4(const Vector& v) const + { + return ((_mm_movemask_ps(GreaterThan(v)) == 0x0f) != 0); + } + + FORCE_INLINE Vector Vector::GreaterThanEqual(const Vector& v) const + { + return _mm_cmpge_ps(m_data, v); + } + + FORCE_INLINE bool Vector::IsAnyGreaterThanEqual(const Vector& v) const + { + return !GreaterThanEqual(v).IsZero4(); + } + + FORCE_INLINE bool Vector::IsGreaterThanEqual2(const Vector& v) const + { + return ((_mm_movemask_ps(GreaterThanEqual(v)) & 3) == 3) != 0; + } + + FORCE_INLINE bool Vector::IsGreaterThanEqual3(const Vector& v) const + { + return ((_mm_movemask_ps(GreaterThanEqual(v)) & 7) == 7) != 0; + } + + FORCE_INLINE bool Vector::IsGreaterThanEqual4(const Vector& v) const + { + return (_mm_movemask_ps(GreaterThanEqual(v)) == 0x0f) != 0; + } + + FORCE_INLINE Vector Vector::LessThan(const Vector& v) const + { + return _mm_cmplt_ps(m_data, v); + } + + FORCE_INLINE bool Vector::IsAnyLessThan(const Vector& v) const + { + return !LessThan(v).IsZero4(); + } + + FORCE_INLINE bool Vector::IsLessThan2(const Vector& v) const + { + return (((_mm_movemask_ps(LessThan(v)) & 3) == 3) != 0); + } + + FORCE_INLINE bool Vector::IsLessThan3(const Vector& v) const + { + return (((_mm_movemask_ps(LessThan(v)) & 7) == 7) != 0); + } + + FORCE_INLINE bool Vector::IsLessThan4(const Vector& v) const + { + return ((_mm_movemask_ps(LessThan(v)) == 0x0f) != 0); + } + + FORCE_INLINE Vector Vector::LessThanEqual(const Vector& v) const + { + return _mm_cmple_ps(m_data, v); + } + + FORCE_INLINE bool Vector::IsAnyLessThanEqual(const Vector& v) const + { + return !LessThanEqual(v).IsZero4(); + } + + FORCE_INLINE bool Vector::IsLessThanEqual2(const Vector& v) const + { + return (((_mm_movemask_ps(LessThanEqual(v)) & 3) == 3) != 0); + } + + FORCE_INLINE bool Vector::IsLessThanEqual3(const Vector& v) const + { + return (((_mm_movemask_ps(LessThanEqual(v)) & 7) == 7) != 0); + } + + FORCE_INLINE bool Vector::IsLessThanEqual4(const Vector& v) const + { + return ((_mm_movemask_ps(LessThanEqual(v)) == 0x0f) != 0); + } + + FORCE_INLINE Vector Vector::EqualsZero() const + { + return Equal(Vector::Zero); + } + + FORCE_INLINE bool Vector::IsAnyEqualToZero2() const + { + return !EqualsZero().IsZero2(); + } + + FORCE_INLINE bool Vector::IsAnyEqualToZero3() const + { + return !EqualsZero().IsZero3(); + } + + FORCE_INLINE bool Vector::IsAnyEqualToZero4() const + { + return !EqualsZero().IsZero4(); + } + + FORCE_INLINE bool Vector::IsZero2() const + { + return IsEqual2(Vector::Zero); + } + + FORCE_INLINE bool Vector::IsZero3() const + { + return IsEqual3(Vector::Zero); + } + + FORCE_INLINE bool Vector::IsZero4() const + { + return IsEqual4(Vector::Zero); + } + + FORCE_INLINE Vector Vector::NearEqualsZero(float epsilon) const + { + return NearEqual(Vector::Zero, Vector(epsilon)); + } + + FORCE_INLINE bool Vector::IsNearZero2(float epsilon) const + { + return IsNearEqual2(Vector::Zero, Vector(epsilon)); + } + + FORCE_INLINE bool Vector::IsNearZero3(float epsilon) const + { + return IsNearEqual3(Vector::Zero, Vector(epsilon)); + } + + FORCE_INLINE bool Vector::IsNearZero4(float epsilon) const + { + return IsNearEqual4(Vector::Zero, Vector(epsilon)); + } + + FORCE_INLINE Vector Vector::EqualsInfinity() const + { + __m128 vTemp = _mm_and_ps(m_data, SIMD::g_absMask); + return _mm_cmpeq_ps(vTemp, Vector::Infinity); + } + + FORCE_INLINE bool Vector::IsInfinite2() const + { + return (_mm_movemask_ps(EqualsInfinity()) & 3) != 0; + } + + FORCE_INLINE bool Vector::IsInfinite3() const + { + return (_mm_movemask_ps(EqualsInfinity()) & 7) != 0; + } + + FORCE_INLINE bool Vector::IsInfinite4() const + { + return (_mm_movemask_ps(EqualsInfinity()) != 0); + } + + FORCE_INLINE Vector Vector::EqualsNaN() const + { + return _mm_cmpneq_ps(m_data, m_data); + } + + FORCE_INLINE bool Vector::IsNaN2() const + { + return (_mm_movemask_ps(EqualsNaN()) & 3) != 0; + } + + FORCE_INLINE bool Vector::IsNaN3() const + { + return (_mm_movemask_ps(EqualsNaN()) & 7) != 0; + } + + FORCE_INLINE bool Vector::IsNaN4() const + { + return (_mm_movemask_ps(EqualsNaN()) != 0); + } + + FORCE_INLINE bool Vector::IsParallelTo(const Vector& v) const + { + Vector const vAbsDot = Vector::Dot3(*this, v).GetAbs(); + Vector const vAbsDelta = Vector::One - vAbsDot; + return vAbsDelta.IsLessThanEqual4(Vector::Epsilon); + } + + FORCE_INLINE void Vector::ToDirectionAndLength2(Vector& direction, float& length) const + { + Vector const vLength = Length2(); + direction = Vector::Select(*this, Vector::Zero, Select0011); + direction /= vLength; + length = vLength.ToFloat(); + } + + FORCE_INLINE void Vector::ToDirectionAndLength3(Vector& direction, float& length) const + { + Vector const vLength = Length3(); + direction = Vector::Select(*this, Vector::Zero, Select0001); + direction /= vLength; + length = vLength.ToFloat(); + } + + FORCE_INLINE bool Vector::operator==(const Vector& rhs) const + { + return IsEqual4(rhs); + } + + FORCE_INLINE bool Vector::operator!=(const Vector& rhs) const + { + return !IsEqual4(rhs); + } +} diff --git a/MotionCorrection/src/cpp/Platform.h b/MotionCorrection/src/cpp/Platform.h new file mode 100644 index 0000000000000000000000000000000000000000..b8569c609f1c6806a06c12c9d5310a670a0cc6f8 --- /dev/null +++ b/MotionCorrection/src/cpp/Platform.h @@ -0,0 +1,51 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +// Finds the current platform +#if defined( __WIN32__ ) || defined( _WIN32 ) +# define PLATFORM_WIN32 +#else +# define PLATFORM_LINUX +#endif + +// +// Platform Specific Helpers/Functions +// + +// DLL export +#if defined(PLATFORM_WIN32) // Windows +# if defined(COMPILER_MSVC) +# if defined(STATIC_LIB) +# define API +# else +# if defined(API) +# define API __declspec(dllexport) +# else +# define API __declspec(dllimport) +# endif +# endif +# else +# if defined(STATIC_LIB) +# define API +# else +# if defined(API) +# define API __attribute__ ((dllexport)) +# else +# define API __attribute__ ((dllimport)) +# endif +# endif +# endif +# define DISABLE_OPTIMIZATION __pragma( optimize( "", off ) ) +# define ENABLE_OPTIMIZATION __pragma( optimize( "", on ) ) +# define DEBUG_BREAK() // __debugbreak() +#else // Linux settings +# include +# define API __attribute__ ((visibility ("default"))) +# define DISABLE_OPTIMIZATION +# define ENABLE_OPTIMIZATION +# define DEBUG_BREAK() // raise(SIGTRAP) +#endif diff --git a/README.md b/README.md index 2c83318c932064b5d90c7ae3d17d06160f75bea6..aaa170d20372b9853e45803f7ce2d4a1ee249a4c 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,278 @@ --- -title: Kimodo Motion Api -emoji: 🏃 -colorFrom: yellow +title: Kimodo Motion API +emoji: 💃 +colorFrom: green colorTo: blue sdk: gradio sdk_version: 6.20.0 -python_version: '3.13' app_file: app.py -pinned: false +python_version: "3.12" +startup_duration_timeout: 1h +short_description: Text-to-motion BVH generation for Blender --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +

+ Banner + License + Project Page + Documentation +

+ +## Overview + +Kimodo is a **ki**nematic **mo**tion **d**iffusi**o**n model trained on a large-scale (700 hours) commercially-friendly optical motion capture dataset. The model generates high-quality 3D human and robot motions, and is controlled through text prompts and an extensive set of constraints such as full-body pose keyframes, end-effector positions/rotations, 2D paths, and 2D waypoints. Full details of the model architecture and training are available in the [technical report](https://research.nvidia.com/labs/sil/projects/kimodo/assets/kimodo_tech_report.pdf). + +This repository provides: +- **Inference**: code and CLI to generate motions on both human and robot skeletons +- **Interactive Demo**: easily author motions with a timeline interface of text prompts and kinematic controls +- **Benchmark**: [test cases](https://huggingface.co/datasets/nvidia/Kimodo-Motion-Gen-Benchmark) and evaluation code built on the [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) dataset to evaluate motion generation models based on text and constraint-following abilities +- **Annotations**: fine-grained temporal text descriptions created for the Kimodo project are included in the [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) dataset. For more information on these labels, see our separate [Hugging Face repo](https://huggingface.co/datasets/nvidia/SEED-Timeline-Annotations). + +### ZeroGPU motion API + +This fork includes a lightweight Hugging Face Gradio app in `app.py`. It exposes a +`/generate_motion` endpoint that generates one SOMA motion and returns Blender-compatible +BVH, Kimodo NPZ, and JSON metadata. The endpoint is designed for ZeroGPU and deliberately +disables the optional native motion-correction pass so the Space does not need to compile +the `MotionCorrection` extension. + +Set the Space hardware to **ZeroGPU** and add an `HF_TOKEN` secret with access to the gated +Llama 3 repositories used by Kimodo's LLM2Vec text encoder. The default model is +`Kimodo-SOMA-RP-v1.1`; override it with the `KIMODO_MODEL` Space variable if needed. + +
+ +
+ +## News + +See the [full changelog](CHANGELOG.md) for a detailed list of all changes. + +- **[2026-07-10]** Released the [ARDY project](https://research.nvidia.com/labs/sil/projects/ardy/) -- a _real-time_ motion generation model with all the controllability of Kimodo! +- **[2026-05-03]** _FIX_: fixed a bug causing incorrect calculation of averaged metrics for constraint test cases in the benchmark +- **[2026-04-24]** _NEW_: improved multi-prompt generation and better support for small VRAM GPUs via `TEXT_ENCODER_DEVICE=cpu` env var +- **[2026-04-10]** Released the [Kimodo Motion Generation Benchmark](#kimodo-motion-generation-benchmark) alongside new v1.1 Kimodo-SOMA models +- **[2026-03-19]** **Breaking:** Model inputs/outputs now use the SOMA 77-joint skeleton (`somaskel77`). +- **[2026-03-16]** Initial open-source release of Kimodo with five model variants (SOMA, G1, SMPL-X), CLI, interactive demo, and timeline annotations for BONES-SEED. + + +## Kimodo Models + +Several variations of Kimodo are available trained on various skeletons and datasets. All models support text-to-motion and kinematic controls. + +> Note: models will be downloaded automatically when attempting to generate from the CLI or Interactive Demo, so there is no need to download them manually + +| Model | Skeleton | Training Data | Release Date | Hugging Face | License | +|:-------|:-------------|:------:|:------:|:-------------:|:-------------:| +| **Kimodo-SOMA-RP-v1.1** | [SOMA](https://github.com/NVlabs/SOMA-X) | [Bones Rigplay 1](https://bones.studio/datasets#rp01) | April 10, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-SOMA-RP-v1.1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-SOMA-SEED-v1.1** | [SOMA](https://github.com/NVlabs/SOMA-X) | [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) | April 10, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-SOMA-SEED-v1.1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-SOMA-RP-v1** | [SOMA](https://github.com/NVlabs/SOMA-X) | [Bones Rigplay 1](https://bones.studio/datasets#rp01) | March 16, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-SOMA-RP-v1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-G1-RP-v1** | [Unitree G1](https://github.com/unitreerobotics/unitree_mujoco/tree/main/unitree_robots/g1) | [Bones Rigplay 1](https://bones.studio/datasets#rp01) | March 16, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-G1-RP-v1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-SOMA-SEED-v1** | [SOMA](https://github.com/NVlabs/SOMA-X) | [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) | March 16, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-SOMA-SEED-v1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-G1-SEED-v1** | [Unitree G1](https://github.com/unitreerobotics/unitree_mujoco/tree/main/unitree_robots/g1) | [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) | March 16, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-G1-SEED-v1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-SMPLX-RP-v1** | [SMPL-X](https://github.com/vchoutas/smplx) | [Bones Rigplay 1](https://bones.studio/datasets#rp01) | March 16, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-SMPLX-RP-v1) | [NVIDIA R&D Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-internal-scientific-research-and-development-model-license/) | + +By default, we recommend using the models trained on the full Bones Rigplay 1 dataset (700 hours of mocap) for your motion generation needs. +The models trained on BONES-SEED use 288 hours of [publicly available mocap data](https://huggingface.co/datasets/bones-studio/seed) so are less capable, but are useful for comparing to other models trained on BONES-SEED. To easily compare motion generation models to Kimodo, check out our [Motion Generation Benchmark](#kimodo-motion-generation-benchmark). + +### Changes in v1.1 +The latest v1.1 Kimodo-SOMA models were released primarily for compatibility with our new [Motion Generation Benchmark](#kimodo-motion-generation-benchmark), but also contain minor quality improvements over v1. For details on these improvements, please see the Hugging Face pages for [Kimodo-SOMA-RP-v1.1](https://huggingface.co/nvidia/Kimodo-SOMA-RP-v1.1#changes-in-v11) and [Kimodo-SOMA-SEED-v1.1](https://huggingface.co/nvidia/Kimodo-SOMA-SEED-v1.1#changes-in-v11). + +## Getting Started + +Please see the full documentation for detailed installation instructions, how to use the CLI and Interactive Demo, and other practical tips for generating motions with Kimodo: + +**[Full Documentation](https://research.nvidia.com/labs/sil/projects/kimodo/docs)** +- [Quick Start Guide](https://research.nvidia.com/labs/sil/projects/kimodo/docs/getting_started/quick_start.html) +- [Installation Instructions](https://research.nvidia.com/labs/sil/projects/kimodo/docs/getting_started/installation.html) +- [Interactive Motion Authoring Demo](https://research.nvidia.com/labs/sil/projects/kimodo/docs/interactive_demo/index.html) +- [Command-Line Interface](https://research.nvidia.com/labs/sil/projects/kimodo/docs/user_guide/cli.html) +- [Benchmark Instructions](https://research.nvidia.com/labs/sil/projects/kimodo/docs/benchmark/introduction.html) +- [API Reference](https://research.nvidia.com/labs/sil/projects/kimodo/docs/api_reference/index.html) + +**Before getting started** with motion generation, please review the [best practices](https://research.nvidia.com/labs/sil/projects/kimodo/docs/key_concepts/limitations.html) and be aware of [model limitations](https://research.nvidia.com/labs/sil/projects/kimodo/docs/key_concepts/limitations.html#limitations). + + +Some notes on installation environment: +- Kimodo requires ~17GB of VRAM to generate locally entirely on GPU, primarily due to the text embedding model. If you have a smaller card, set `TEXT_ENCODER_DEVICE=cpu` when running Kimodo commands to force text encoding to the CPU. This is slightly slower but reduces VRAM usage to <3 GB. +- The model has been most extensively tested on GeForce RTX 3090, GeForce RTX 4090, and NVIDIA A100 GPUs, but should work on other recent cards with sufficient VRAM +- This repo was developed on Linux, though Windows should work especially if using Docker + +## Interactive Motion Authoring Demo + +
+ +
+ +
+ +**[Demo Documentation and Tutorial](https://research.nvidia.com/labs/sil/projects/kimodo/docs/interactive_demo/index.html)** + +The web-based interactive demo provides an intuitive interface for generating motions with any of the Kimodo model variations. After installation, the demo can be launched with the `kimodo_demo` command. It runs locally on http://127.0.0.1:7860. Open this URL in your browser to access the interface (or use port forwarding if set up on a server). + +### Demo Features +- **Multiple Characters**: Supports generating with the SOMA, G1, and SMPL-X versions of Kimodo +- **Text Prompts**: Enter one or more natural language descriptions of desired motions on the timeline +- **Timeline Editor**: Add and edit keyframes and constrained intervals on multiple constraint tracks +- **Constraint Types**: + - Full-Body: Complete joint position constraints at specific frames + - 2D Root: Define waypoints or full paths to follow on the ground plane + - End-Effectors: Control hands and feet positions/rotations +- **Constraint Editing**: Editing mode allows for re-posing of constraints or adjusting waypoints +- **3D Visualization**: Real-time rendering of generated motions with skeleton and skinned mesh options +- **Playback Controls**: Preview generated motions with adjustable playback speed +- **Multiple Samples**: Generate and compare multiple motion variations +- **Examples**: Load pre-existing examples to better understand Kimodo's capabilities +- **Export**: Save constraints and generated motions for later use + +## Command-Line Interface + +**[CLI Documentation and Examples](https://research.nvidia.com/labs/sil/projects/kimodo/docs/user_guide/cli.html)** + +Motions can also be generated directly from the command line with the `kimodo_gen` command or by running `python -m kimodo.scripts.generate` directly. + +**Key Arguments:** +- `prompt`: A single text description or sequence of texts for the desired motion (required) +- `--model`: Which Kimodo model to use for generation +- `--duration`: Motion duration in seconds +- `--num_samples`: Number of motion variations to generate +- `--constraints`: Constraint file to control the generated motion (e.g., saved from the web demo) +- `--diffusion_steps`: Number of denoising steps +- `--cfg_type` / `--cfg_weight`: Classifier-free guidance (`nocfg`, `regular` with one weight, or `separated` with two weights for text vs. constraints); see the [CLI docs](https://research.nvidia.com/labs/sil/projects/kimodo/docs/user_guide/cli.html#classifier-free-guidance-cfg) +- `--no-postprocess`: Flag to disable foot skate and constraint cleanup post-processing +- `--seed`: Random seed for reproducible results + +The script supports different output formats depending on which skeleton is used. By default, a custom NPZ format is saved that is compatible with the web demo. +For Kimodo-G1 models, the motion can be saved in the standard MuJoCo qpos CSV format. +For Kimodo-SMPLX, motion can be saved in the standard AMASS npz format for compability with existing pipelines. + +### Default NPZ Output Format +Generated motions are saved as NPZ files containing: +- `posed_joints`: Global joint positions `[T, J, 3]` +- `global_rot_mats`: Global joint rotation matrices `[T, J, 3, 3]` +- `local_rot_mats`: Local (parent-relative) joint rotation matrices `[T, J, 3, 3]` +- `foot_contacts`: Foot contact labels [left heel, left toe, right heel, right toes] `[T, 4]` +- `smooth_root_pos`: Smoothed root representations outputted from the model `[T, 3]` +- `root_positions`: The (non-smoothed) trajectory of the actual root joint (e.g., pelvis) `[T, 3]` +- `global_root_heading`: The heading direction output from the model `[T, 2]` + +`T` the number of frames and `J` the number of joints. + +## Low-Level Python API + +**[Model API Documentation](https://research.nvidia.com/labs/sil/projects/kimodo/docs/api_reference/model.html#kimodo.model.kimodo_model.Kimodo.__call__)** + +For maximum flexibility, the low-level model inference API can be called directly, rather than going through our high-level CLI. +This allows for advanced model configuration including classifier-free guidance weights and parameters related to transitions in multi-prompt sequences. + +## Downstream Robotics Applications of Kimodo + +### Visualizing G1 Motions with MuJoCo + +
+ +
+ +After generating motions on the G1 robot skeleton and saving to the MuJoCo qpos CSV file format, they can be easily used and visualized within MuJoCo. +A minimal visualization script is available with: +``` +python -m kimodo.scripts.mujoco_load +``` +Make sure to edit the script to correctly point to your CSV file and install Mujoco before running this. + +### Tracking Generated Motions with ProtoMotions + +
+ +
+ +[ProtoMotions](https://github.com/NVlabs/ProtoMotions) is a GPU-accelerated simulation and learning framework for training physically simulated digital humans and humanoid robots. The Kimodo NPZ and CSV output formats are both compatible with ProtoMotions making it easy to train physics-based policies with generated motions from Kimodo. ProtoMotions supports outputs on both the SOMA skeleton and Unitree G1 + +After generating motions with Kimodo, head over to the [ProtoMotions docs](https://github.com/NVlabs/ProtoMotions?tab=readme-ov-file#-motion-authoring-with-kimodo) to see how to import them. + +### Retargeting Motions to Other Robots with GMR + +
+ +
+ +Motions generated by Kimodo-SMPLX can be retargeted to other robots using [General Motion Retargeting (GMR)](https://github.com/YanjieZe/GMR). +GMR supports the AMASS NPZ format out of the box, so simply generate motions with Kimodo and use `--output` to save; the AMASS NPZ is written to `stem_amass.npz` (single sample) or in the output folder (multiple samples). Then, use the [SMPL-X to Robot script](https://github.com/YanjieZe/GMR?tab=readme-ov-file#retargeting-from-smpl-x-amass-omomo-to-robot) in GMR to retarget to any supported robot. For example: +``` +# run within GMR codebase +python scripts/smplx_to_robot.py --smplx_file /path/to/saved/amass_format.npz --robot booster_t1 +``` + +### Combining Kimodo with GEAR-SONIC + +
+ +
+ +As a proof of concept, we have also incorporated Kimodo into the [interactive GEAR-SONIC demo](https://nvlabs.github.io/GEAR-SONIC/demo.html). In the demo, Kimodo can be used to generate a kinematic motion on the G1 robot skeleton, then GEAR-SONIC tracks the motion in simulation. + +## Kimodo Motion Generation Benchmark + +[**[Benchmark Documentation](https://research.nvidia.com/labs/sil/projects/kimodo/docs/benchmark/introduction.html)**] +[**[Test Suite on Hugging Face](https://huggingface.co/datasets/nvidia/Kimodo-Motion-Gen-Benchmark)**] + +Alongside the Kimodo models, we provide a benchmark designed to standardize evaluation for motion generation models with a comprehensive set of test cases. This includes: + +* **Evaluation Data**: A suite of test cases [available on Hugging Face](https://huggingface.co/datasets/nvidia/Kimodo-Motion-Gen-Benchmark) is used in concert with the [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) dataset to construct the full benchmark. +* **Diverse Test Cases**: Test cases cover a wide range of text-conditioned and constraint-conditioned motion generation. +* **Evaluation Pipeline**: Code for the full evaluation pipeline including benchmark construction, motion generation, and evaluation. +* **Metrics**: Several metrics to evaluate generated motions that cover motion quality, constraint following, and text alignment. Our [TMR-SOMA-RP-v1](https://huggingface.co/nvidia/TMR-SOMA-RP-v1) model trained on all 700 hours of the Bones Rigplay dataset is a powerful embedding model to compute common metrics like R-precision and FID. + +To facilitate future research, we [report benchmark results](https://research.nvidia.com/labs/sil/projects/kimodo/docs/benchmark/results.html) for Kimodo-SOMA-v1.1 models, which are reproducible and easily comparable to other methods trained on the BONES-SEED data. + +## Timeline Annotations for BONES-SEED + +As detailed in the [tech report](https://research.nvidia.com/labs/sil/projects/kimodo/assets/kimodo_tech_report.pdf), Kimodo is trained using fine-grained temporal text annotations of mocap clips. +While the full [Rigplay 1](https://bones.studio/datasets#rp01) dataset is proprietary, we have released the temporal segmentations for the public [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) subset. +These annotations are already included in the BONES-SEED dataset, but the standalone labels and additional information about them is [available on HuggingFace](https://huggingface.co/datasets/nvidia/SEED-Timeline-Annotations). + + +## Related Humanoid Work at NVIDIA +Kimodo is part of a larger effort to enable humanoid motion data for robotics, physical AI, and other applications. + +Check out these related works: +* [ARDY](https://github.com/nv-tlabs/ardy) - builds on top of Kimodo to enable real-time controllable motion generation for interactive applications +* [MotionBricks](https://nvlabs.github.io/motionbricks/) - real-time motion generation framework that specializes in fast and robust motion in-betweening +* [SOMA Body Model](https://github.com/NVlabs/SOMA-X) - a unified parameteric human body model +* [BONES-SEED Dataset](https://huggingface.co/datasets/bones-studio/seed) - a large scale human(oid) motion capture dataset in SOMA and G1 format +* [ProtoMotions](https://github.com/NVlabs/ProtoMotions) - simulation and learning framework for training physically simulated human(oid)s +* [SOMA Retargeter](https://github.com/NVIDIA/soma-retargeter) - SOMA to G1 retargeting tool +* [GEM](https://github.com/NVlabs/GEM-X) - human motion reconstruction from video +* [GEAR SONIC](https://github.com/NVlabs/GR00T-WholeBodyControl) - humanoid behavior foundation model for physical robots + +## Citation + +If you use this code in your research, please cite: + +```bibtex +@article{Kimodo2026, + title={Kimodo: Scaling Controllable Human Motion Generation}, + author={Rempe, Davis and Petrovich, Mathis and Yuan, Ye and Zhang, Haotian and Peng, Xue Bin and Jiang, Yifeng and Wang, Tingwu and Iqbal, Umar and Minor, David and de Ruyter, Michael and Li, Jiefeng and Tessler, Chen and Lim, Edy and Jeong, Eugene and Wu, Sam and Hassani, Ehsan and Huang, Michael and Yu, Jin-Bey and Chung, Chaeyeon and Song, Lina and Dionne, Olivier and Kautz, Jan and Yuen, Simon and Fidler, Sanja}, + journal={arXiv:2603.15546}, + year={2026} +} +``` + +## License + +This codebase is licensed under [Apache-2.0](LICENSE). Note that model checkpoints and data are licensed separately as indicated on the HuggingFace download pages. + +This project will download and install additional third-party open source software projects. Review the license terms of these open source projects before use. + +## Acknowledgments + +This project builds upon excellent open-source projects: +- [Viser](https://github.com/nerfstudio-project/viser) for 3D motion authoring demo +- [LLM2Vec](https://github.com/McGill-NLP/llm2vec) for text encoding + +## Contact + +For questions or issues, please open an issue on this repository or reach out directly to the authors. + +--- + diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..ff9a58f085a35632198654cd0e71b43d3b2f0f99 --- /dev/null +++ b/app.py @@ -0,0 +1,172 @@ +"""Minimal Kimodo text-to-motion API for Hugging Face ZeroGPU.""" + +from __future__ import annotations + +import os +import tempfile +from pathlib import Path + +os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") +os.environ.setdefault("TEXT_ENCODER_MODE", "local") +os.environ.setdefault("TEXT_ENCODER_DEVICE", "cuda") + +# ZeroGPU must patch CUDA before torch or any Kimodo module imports it. +import spaces +import gradio as gr +import torch + +from kimodo import load_model +from kimodo.exports.bvh import save_motion_bvh +from kimodo.exports.motion_io import save_kimodo_npz +from kimodo.tools import seed_everything +from space_utils import estimate_zero_gpu_duration, validate_motion_request + + +MODEL_NAME = os.environ.get("KIMODO_MODEL", "Kimodo-SOMA-RP-v1.1") + +# Eager module-scope placement is required by ZeroGPU's weight packing mechanism. +MODEL, RESOLVED_MODEL_NAME = load_model( + MODEL_NAME, + device="cuda", + default_family="Kimodo", + return_resolved_name=True, +) + + +def _without_progress(iterable): + return iterable + + +def _single_sample(output: dict) -> dict: + """Remove the leading batch dimension from a one-sample model result.""" + + sample = {} + for key, value in output.items(): + if hasattr(value, "shape") and len(value.shape) > 0 and int(value.shape[0]) == 1: + sample[key] = value[0] + else: + sample[key] = value + return sample + + +@spaces.GPU(duration=estimate_zero_gpu_duration, size="large") +def generate_motion( + prompt: str, + duration_seconds: float, + seed: int, + diffusion_steps: int, + standard_tpose: bool, +) -> tuple[str, str, dict]: + """Generate one text-conditioned human motion and return BVH, NPZ, and metadata.""" + + request = validate_motion_request( + prompt, + duration_seconds, + seed, + diffusion_steps, + standard_tpose, + ) + seed_everything(request.seed) + + num_frames = max(1, int(round(request.duration_seconds * float(MODEL.fps)))) + output = MODEL( + request.prompt, + num_frames, + num_denoising_steps=request.diffusion_steps, + multi_prompt=False, + constraint_lst=[], + cfg_weight=[2.0, 2.0], + num_samples=1, + return_numpy=True, + post_processing=False, + progress_bar=_without_progress, + ) + sample = _single_sample(output) + + output_dir = Path(tempfile.mkdtemp(prefix="kimodo-motion-")) + bvh_path = output_dir / "motion.bvh" + npz_path = output_dir / "motion.npz" + save_kimodo_npz(str(npz_path), sample) + + skeleton = MODEL.output_skeleton + local_rot_mats = torch.as_tensor(sample["local_rot_mats"], device="cuda") + posed_joints = torch.as_tensor(sample["posed_joints"], device="cuda") + root_positions = posed_joints[:, int(skeleton.root_idx), :] + save_motion_bvh( + bvh_path, + local_rot_mats, + root_positions, + skeleton=skeleton, + fps=float(MODEL.fps), + standard_tpose=request.standard_tpose, + ) + + metadata = { + "prompt": request.prompt, + "duration_seconds": request.duration_seconds, + "seed": request.seed, + "diffusion_steps": request.diffusion_steps, + "model": MODEL_NAME, + "resolved_model": RESOLVED_MODEL_NAME, + "fps": float(MODEL.fps), + "frames": num_frames, + "skeleton": str(skeleton.name), + "standard_tpose": request.standard_tpose, + "post_processing": False, + } + return str(bvh_path), str(npz_path), metadata + + +with gr.Blocks(title="Kimodo Motion API") as demo: + gr.Markdown( + """ + # Kimodo Motion API + + Generate a text-conditioned SOMA motion on ZeroGPU. Download the BVH for Blender + or keep the NPZ for a later Kimodo workflow. The first call after an idle period may + take longer while ZeroGPU restores model weights. + """ + ) + prompt_input = gr.Textbox( + label="Motion prompt", + lines=3, + max_length=1_000, + value="A person walks forward cautiously, looks over the left shoulder, then stops.", + ) + with gr.Row(): + duration_input = gr.Slider(1.0, 10.0, value=5.0, step=0.5, label="Duration (seconds)") + seed_input = gr.Number(value=42, precision=0, minimum=0, maximum=2**31 - 1, label="Seed") + steps_input = gr.Slider(10, 100, value=50, step=5, label="Diffusion steps") + standard_tpose_input = gr.Checkbox( + value=True, + label="Export a standard T-pose rest skeleton", + info="Recommended for Blender retargeting.", + ) + generate_button = gr.Button("Generate motion", variant="primary") + with gr.Row(): + bvh_output = gr.File(label="Blender BVH") + npz_output = gr.File(label="Kimodo NPZ") + metadata_output = gr.JSON(label="Generation metadata") + + generate_button.click( + fn=generate_motion, + inputs=[prompt_input, duration_input, seed_input, steps_input, standard_tpose_input], + outputs=[bvh_output, npz_output, metadata_output], + api_name="generate_motion", + api_description="Generate a Kimodo motion as BVH and NPZ files.", + concurrency_limit=1, + ) + gr.Examples( + examples=[ + ["A person takes three slow steps forward and waves with the right hand."], + ["A person crouches, jumps upward, lands, and regains balance."], + ["A person performs a short defensive boxing combination."], + ], + inputs=[prompt_input], + cache_examples=False, + ) + +demo.queue(default_concurrency_limit=1) + +if __name__ == "__main__": + demo.launch(mcp_server=True) diff --git a/assets/banner.png b/assets/banner.png new file mode 100644 index 0000000000000000000000000000000000000000..47a0b54f3176839cb79e426fa09c5d72a326d5c3 Binary files /dev/null and b/assets/banner.png differ diff --git a/assets/demo_screenshot.png b/assets/demo_screenshot.png new file mode 100644 index 0000000000000000000000000000000000000000..71ed6db85adf7c2b27ecb0d38313b6862f157296 --- /dev/null +++ b/assets/demo_screenshot.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f30c5ff61905841d7104ebecb9ca6321426cb383eaae32a1e6676adb1fc1442d +size 469052 diff --git a/assets/gmr_results.gif b/assets/gmr_results.gif new file mode 100644 index 0000000000000000000000000000000000000000..fdccd1301a4357fef7afe0d79007ae084990ed1d --- /dev/null +++ b/assets/gmr_results.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76c317885ef2eee240c4859e1b98aca81ef5ae8cc8c0a140c1a0c44230b075cc +size 1855914 diff --git a/assets/mujoco_result.gif b/assets/mujoco_result.gif new file mode 100644 index 0000000000000000000000000000000000000000..a5937dd68f09f12756e8da19d3ffe8acc83eebe8 --- /dev/null +++ b/assets/mujoco_result.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4988ec1722fa92b3154a305bfcc1c6b3d922e31f37d26f4ad11be4eaa481d07c +size 1854599 diff --git a/assets/protomotions_results.gif b/assets/protomotions_results.gif new file mode 100644 index 0000000000000000000000000000000000000000..95f8c426132688d09bcf13d0c473068c1348a20a --- /dev/null +++ b/assets/protomotions_results.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee3145cf4bfa83036992a2bd953354622eba1345aece87653b3183ab11cc8890 +size 3881892 diff --git a/assets/sonic_kimodo_demo.gif b/assets/sonic_kimodo_demo.gif new file mode 100644 index 0000000000000000000000000000000000000000..3c375f9af1926c01090053aa3d4e672b69c54f1d --- /dev/null +++ b/assets/sonic_kimodo_demo.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99b3978ffa56180e14cef6c84e10a8041c512e6b882c5ade4b0b3e02882b739b +size 1811888 diff --git a/assets/teaser.gif b/assets/teaser.gif new file mode 100644 index 0000000000000000000000000000000000000000..670f084da04caf703fc1c25c5b4fbed725541236 --- /dev/null +++ b/assets/teaser.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:746ac8540643147686bb4148a6d39401debcef50a6ea13ba5069a5f6405a9ea7 +size 3853319 diff --git a/benchmark/create_benchmark.py b/benchmark/create_benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..810a7ca8f7b40d240a43aecd872250bd7cebe56a --- /dev/null +++ b/benchmark/create_benchmark.py @@ -0,0 +1,188 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Step (1) of evaluation pipeline. + +This script builds the benchmark test suites from BVH motions in the Bones-SEED dataset using +the benchmark metadata. Currently it is only set up for the SOMA skeleton. +""" + +import argparse +from functools import partial +from multiprocessing import Pool +from pathlib import Path + +import numpy as np +import torch +from tqdm import tqdm + +from kimodo.geometry import matrix_to_axis_angle +from kimodo.motion_rep import KimodoMotionRep +from kimodo.skeleton import SOMASkeleton77 +from kimodo.skeleton.bvh import parse_bvh_motion +from kimodo.tools import load_json, save_json, to_numpy, to_torch + +FPS = 30 +BENCHMARK_REPO_ID = "nvidia/Kimodo-Motion-Gen-Benchmark" + + +def download_benchmark(dest: Path) -> Path: + """Download the benchmark testsuite from HuggingFace to *dest*.""" + from huggingface_hub import snapshot_download + + print(f"Downloading benchmark testsuite from {BENCHMARK_REPO_ID} to {dest} ...") + snapshot_dir = snapshot_download( + repo_id=BENCHMARK_REPO_ID, + repo_type="dataset", + local_dir=str(dest), + ) + return Path(snapshot_dir) + + +def discover_seed_motion_folders(root: Path) -> list[Path]: + """Find all directories under root that contain seed_motion.json; return sorted list of those + dirs.""" + root = root.resolve() + if not root.is_dir(): + raise FileNotFoundError(f"Folder does not exist: {root}") + out: list[Path] = [] + for meta_path in root.rglob("seed_motion.json"): + src_dir = meta_path.parent + out.append(src_dir) + return sorted(out) + + +def constraints_and_motion_from_seed(folder: str, dataset_folder: str, fps=FPS): + """Load seed_motion.json and BVH from folder; subsample to fps, convert to SOMA gt_motion.npz + and constraints.""" + folder = Path(folder) + dataset_folder = Path(dataset_folder) + out_path = folder / "gt_motion.npz" + + seed_motion = load_json(folder / "seed_motion.json") + + start = seed_motion["crop_start_frame_index"] + end = seed_motion["crop_end_frame_index"] + + bvh_path = dataset_folder / seed_motion["bvh_path"].replace("BVH/", "bvh/") + + local_rot_mats, root_trans, bvh_fps = parse_bvh_motion(bvh_path) + step = round(bvh_fps / fps) + + # Subsample fps + root_trans = root_trans[::step] + local_rot_mats = local_rot_mats[::step] + + skeleton = SOMASkeleton77() + # Changing t_pose: essential step + local_rot_mats, global_rot_mats = skeleton.to_standard_tpose(local_rot_mats) + + # Use the motion rep to canonicalize the motion (start z+ at 0,0) + # and get other components (smooth root, foot contacts etc) + motion_rep = KimodoMotionRep(skeleton, fps) + feats = motion_rep(local_rot_mats, root_trans, to_normalize=False) + + # Crop the features and canonicalizing them + feats = feats[start:end] + can_feats = motion_rep.canonicalize(feats) + # Get back the motion + motion = motion_rep.inverse(can_feats, is_normalized=False) + motion = to_numpy(to_torch(motion, dtype=torch.float32)) + + np.savez(out_path, **motion) + + seed_constraints_path = folder / "seed_constraints.json" + if seed_constraints_path.exists(): + seed_constraints_lst = load_json(seed_constraints_path) + + constraints_lst = [] + for seed_cons in seed_constraints_lst: + cons = seed_cons.copy() + frame_indices = cons["frame_indices"] + + cons["smooth_root_2d"] = motion["smooth_root_pos"][frame_indices][..., [0, 2]].tolist() + + if cons["type"] == "root2d": + if cons.get("use_global_orient", False): + cons["global_root_heading"] = motion["global_root_heading"][ # noqa + frame_indices + ].tolist() + elif cons["type"] in ["fullbody"] or cons["type"] in [ + "left-hand", + "right-hand", + "left-foot", + "right-foot", + "end-effector", + ]: + cons["local_joints_rot"] = matrix_to_axis_angle( + to_torch(motion["local_rot_mats"][frame_indices]) + ).tolist() + cons["root_positions"] = motion["root_positions"][frame_indices].tolist() + else: + raise TypeError(f"This constraint type is not recognized: {cons['type']}") + + constraints_lst.append(cons) + + # check that it is close to old_constraints_lst + save_json(folder / "constraints.json", constraints_lst) + + +def main(): + parser = argparse.ArgumentParser( + description="Recursively find test case to fill with motions and constraints.", + ) + parser.add_argument( + "benchmark", + type=Path, + help="Root folder to search recursively or seed_motion.json for to download the benchmark testsuite from HuggingFace to.", + ) + parser.add_argument( + "--dataset", + type=Path, + default="datasets/bones-seed/soma_uniform", + help="SEED dataset folder", + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Redo the process even if gt_motion.npz already exists", + ) + parser.add_argument( + "--workers", + type=int, + default=1, + help="Number of parallel worker processes (default: 1, sequential)", + ) + args = parser.parse_args() + + folder = args.benchmark.resolve() + if not folder.is_dir(): + print(f"Benchmark folder not found at {folder}, downloading from HuggingFace...") + download_benchmark(folder) + + dirs = discover_seed_motion_folders(folder) + if not dirs: + raise SystemExit(f"No directories with seed_motion.json found under {folder}") + print(f"Discovered {len(dirs)} motion to populate.") + + skipped = 0 + to_process = [] + for d in dirs: + if not args.overwrite and (d / "gt_motion.npz").is_file(): + skipped += 1 + else: + to_process.append(d) + + fn = partial(constraints_and_motion_from_seed, dataset_folder=args.dataset) + with Pool(args.workers) as pool: + list(tqdm(pool.imap_unordered(fn, to_process), total=len(to_process), desc="Extracting GT motions")) + + if skipped: + print(f"Processed {len(dirs) - skipped} folders, skipped {skipped} (already present).") + else: + print("Saved gt_motion.npz and constraints.json from the seed files.") + + +if __name__ == "__main__": + main() diff --git a/benchmark/embed_folder.py b/benchmark/embed_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..5f493bc3cf1dfb9ad6d25c2ab108cad56261602f --- /dev/null +++ b/benchmark/embed_folder.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Step (3) of evaluation pipeline. + +This script recursively embeds generated motions, ground-truth motions, and text prompts from a test suite folder tree with the pre-trained TMR model. +""" + +import argparse +from pathlib import Path + +import numpy as np +import torch +from tqdm import tqdm + +from kimodo.meta import parse_prompts_from_meta +from kimodo.model.load_model import load_model +from kimodo.tools import load_json + + +def discover_motion_folders(root: Path) -> list[Path]: + root = root.resolve() + if not root.is_dir(): + raise FileNotFoundError(f"Folder does not exist: {root}") + out: list[Path] = [] + for meta_path in root.rglob("meta.json"): + src_dir = meta_path.parent + if (src_dir / "motion.npz").is_file() or (src_dir / "gt_motion.npz").is_file(): + out.append(src_dir) + return sorted(out) + + +def _load_posed_joints(npz_path: Path, device: str) -> torch.Tensor: + data = np.load(npz_path) + if "posed_joints" not in data: + raise SystemExit(f"NPZ must contain 'posed_joints': {npz_path}") + posed_joints = data["posed_joints"] + if posed_joints.ndim == 4: + if posed_joints.shape[0] != 1: + raise SystemExit(f"Expected batch size 1 for posed_joints, got {posed_joints.shape[0]} in {npz_path}") + posed_joints = posed_joints[0] + if posed_joints.ndim != 3: + raise SystemExit(f"Expected posed_joints shape [T, J, 3], got {posed_joints.shape} in {npz_path}") + return torch.from_numpy(posed_joints).float().to(device) + + +def main(): + parser = argparse.ArgumentParser( + description="Recursively embed motion, gt_motion, and text; save motion_embedding.npy, gt_motion_embedding.npy, and text_embedding.npy when present.", + ) + parser.add_argument( + "folder", + type=Path, + help="Root folder to search recursively for meta.json and motion.npz and/or gt_motion.npz", + ) + parser.add_argument( + "--model", + default="tmr-soma-rp", + help="Model for encoding (e.g. TMR-SOMA-RP-v1, tmr-soma-rp). Default: tmr-soma-rp", + ) + parser.add_argument( + "--device", + default=None, + help="Device (default: cuda if available else cpu)", + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Re-embed even if embedding files already exist", + ) + parser.add_argument( + "--text_encoder_fp32", + action="store_true", + help="Uses fp32 for the text encoder rather than default bfloat16.", + ) + args = parser.parse_args() + + folder = args.folder.resolve() + if not folder.is_dir(): + raise SystemExit(f"Folder does not exist or is not a directory: {folder}") + + device = args.device or ("cuda" if torch.cuda.is_available() else "cpu") + model = load_model(modelname=args.model, device=device, default_family="TMR", text_encoder_fp32=args.text_encoder_fp32) + + dirs = discover_motion_folders(folder) + if not dirs: + raise SystemExit(f"No directories with meta.json and (motion.npz or gt_motion.npz) found under {folder}") + print(f"Discovered {len(dirs)} motion folders.") + + skipped_motion = 0 + skipped_gt = 0 + skipped_text = 0 + for sample_dir in tqdm(dirs, desc="Embedding"): + meta_path = sample_dir / "meta.json" + meta = load_json(meta_path) + texts, _ = parse_prompts_from_meta(meta) + if len(texts) != 1: + raise SystemExit(f"Expected exactly one text per motion; got {len(texts)} in {meta_path}") + text = texts[0] + + # Embed motion.npz -> motion_embedding.npy + if (sample_dir / "motion.npz").is_file(): + if not args.overwrite and (sample_dir / "motion_embedding.npy").is_file(): + skipped_motion += 1 + else: + npz_path = sample_dir / "motion.npz" + posed_joints = _load_posed_joints(npz_path, device) + with torch.inference_mode(): + motion_emb = model.encode_motion(posed_joints, unit_vector=True) + np.save(sample_dir / "motion_embedding.npy", motion_emb.cpu().numpy()) + + # Embed gt_motion.npz -> gt_motion_embedding.npy + if (sample_dir / "gt_motion.npz").is_file(): + if not args.overwrite and (sample_dir / "gt_motion_embedding.npy").is_file(): + skipped_gt += 1 + else: + npz_path = sample_dir / "gt_motion.npz" + posed_joints = _load_posed_joints(npz_path, device) + with torch.inference_mode(): + gt_motion_emb = model.encode_motion(posed_joints, unit_vector=True) + np.save(sample_dir / "gt_motion_embedding.npy", gt_motion_emb.cpu().numpy()) + + # Embed text -> text_embedding.npy + if not args.overwrite and (sample_dir / "text_embedding.npy").is_file(): + skipped_text += 1 + else: + with torch.inference_mode(): + text_emb = model.encode_raw_text([text], unit_vector=True) + np.save(sample_dir / "text_embedding.npy", text_emb.cpu().numpy()) + + total_skipped = skipped_motion + skipped_gt + skipped_text + if total_skipped: + print(f"Embedded {len(dirs)} folders; skipped some existing files (use --overwrite to re-embed).") + else: + print(f"Saved motion_embedding.npy, gt_motion_embedding.npy, and text_embedding.npy in {len(dirs)} folders.") + + +if __name__ == "__main__": + main() diff --git a/benchmark/evaluate_folder.py b/benchmark/evaluate_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..aa1deeae5f726c0ee711fa7cb7ec006997e21e61 --- /dev/null +++ b/benchmark/evaluate_folder.py @@ -0,0 +1,359 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Step (4) of evaluation pipeline. + +This script recursively computes metrics for generated and ground-truth motions within a test suite folder tree. +Saves metrics json files per test case and per group of test cases in the folder tree. +""" + +import argparse +import json +from itertools import groupby +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from tqdm import tqdm + +from kimodo.constraints import load_constraints_lst +from kimodo.meta import parse_prompts_from_meta +from kimodo.metrics import ( + ContraintFollow, + FootContactConsistency, + FootSkateFromContacts, + FootSkateFromHeight, + FootSkateRatio, + TMR_EmbeddingMetric, + aggregate_metrics, + clear_metrics, + compute_metrics, + compute_tmr_per_sample_retrieval, +) +from kimodo.skeleton import build_skeleton +from kimodo.skeleton.definitions import SOMASkeleton30 +from kimodo.tools import load_json, to_torch + +DEFAULT_FPS = 30.0 + + +def discover_motion_folders(root: Path) -> list[tuple[Path, Path]]: + root = root.resolve() + if not root.is_dir(): + raise FileNotFoundError(f"Folder does not exist: {root}") + out: list[tuple[Path, Path]] = [] + for meta_path in root.rglob("meta.json"): + sample_dir = meta_path.parent + if (sample_dir / "motion.npz").is_file() and (sample_dir / "gt_motion.npz").is_file(): + rel = sample_dir.relative_to(root) + out.append((sample_dir, rel)) + return sorted(out, key=lambda x: str(x[1])) + + +def group_by_parent(examples: list[tuple[Path, Path]]) -> list[list[tuple[Path, Path]]]: + def parent_key(item: tuple[Path, Path]) -> Path: + return item[1].parent if len(item[1].parts) > 1 else Path(".") + + sorted_examples = sorted(examples, key=parent_key) + groups: list[list[tuple[Path, Path]]] = [] + for _key, group in groupby(sorted_examples, key=parent_key): + groups.append(list(group)) + return groups + + +def _to_scalar(t: torch.Tensor) -> float: + return float(t.mean().item()) if t.numel() > 0 else float(t.item()) + + +def _to_p95(t: torch.Tensor) -> float: + if t.numel() == 0: + return float("nan") + return float(torch.nanquantile(t, torch.tensor(0.95, device=t.device), dim=0).item()) + + +def _per_sample_metrics_from_saved(metrics_list: list, n: int) -> list[dict[str, float]]: + per_sample: list[dict[str, float]] = [{} for _ in range(n)] + for metric in metrics_list: + for key, lst in metric.saved_metrics.items(): + for i, t in enumerate(lst): + if i >= n: + break + per_sample[i][key] = _to_scalar(t) + return per_sample + + +def _load_pair_embeddings( + sample_dir: Path, +) -> tuple[np.ndarray, np.ndarray, np.ndarray | None] | None: + motion_emb_path = sample_dir / "motion_embedding.npy" + text_emb_path = sample_dir / "text_embedding.npy" + gt_motion_emb_path = sample_dir / "gt_motion_embedding.npy" + if not (motion_emb_path.is_file() and text_emb_path.is_file()): + return None + + motion_emb = np.load(motion_emb_path) + text_emb = np.load(text_emb_path) + if motion_emb.ndim == 3 and motion_emb.shape[0] == 1: + motion_emb = motion_emb[0] + if text_emb.ndim == 3 and text_emb.shape[0] == 1: + text_emb = text_emb[0] + + gt_motion_emb = None + if gt_motion_emb_path.is_file(): + gt_motion_emb = np.load(gt_motion_emb_path) + if gt_motion_emb.ndim == 3 and gt_motion_emb.shape[0] == 1: + gt_motion_emb = gt_motion_emb[0] + + return motion_emb, text_emb, gt_motion_emb + + +def _load_npz_motion( + npz_path: Path, + device: str, + soma30_skel: SOMASkeleton30 | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Load posed_joints and foot_contacts from an NPZ, upscaling SOMA30 to SOMA77 if needed.""" + data = np.load(npz_path) + posed_joints = to_torch(data["posed_joints"], device=device) + foot_contacts = to_torch(data["foot_contacts"], device=device) + + if posed_joints.shape[-2] == 30 and soma30_skel is not None: + local_rot_mats = to_torch(data["local_rot_mats"], device=device) + root_positions = to_torch(data["root_positions"], device=device) + out77 = soma30_skel.output_to_SOMASkeleton77( + {"local_rot_mats": local_rot_mats, "root_positions": root_positions, "foot_contacts": foot_contacts} + ) + posed_joints = out77["posed_joints"] + foot_contacts = out77["foot_contacts"] + + return posed_joints, foot_contacts + + +def _run_eval_on_group( + group: list[tuple[Path, Path]], + skeleton: torch.nn.Module, + metrics_list: list, + device: str, + group_name: str = "", + soma30_skel: SOMASkeleton30 | None = None, +) -> tuple[ + list[dict[str, float]], + list[dict[str, float]], + dict[str, float], + dict[str, float], + dict[str, float], + list[dict[str, Any]], +]: + """Run two passes: gen (motion.npz + embeddings) and GT (gt_motion.npz only). Return + per_sample_gen, per_sample_gt, aggregated_gen, aggregated_gt, tmr_metrics, tmr_per_sample. + """ + n = len(group) + sample_ids: list[str] = [] + texts: list[str] = [] + motion_embs: list[np.ndarray] = [] + text_embs: list[np.ndarray] = [] + + # ----- Pass 1: generation (motion.npz + all embeddings) ----- + clear_metrics(metrics_list) + desc = f"Samples ({group_name})" if group_name else "Samples" + for sample_dir, rel_path in tqdm(group, desc=desc, unit="motion"): + stem = rel_path.name + sample_ids.append(stem) + meta_path = sample_dir / "meta.json" + meta = load_json(meta_path) + texts_parsed, _ = parse_prompts_from_meta(meta) + texts.append(texts_parsed[0] if texts_parsed else "") + + posed_joints, foot_contacts = _load_npz_motion(sample_dir / "motion.npz", device, soma30_skel) + nframes = posed_joints.shape[0] + lengths = torch.tensor(nframes, dtype=torch.long, device=device) + constraints_path = sample_dir / "constraints.json" + constraints_lst = ( + load_constraints_lst(str(constraints_path), skeleton=skeleton) if constraints_path.is_file() else [] + ) + metrics_in: dict[str, Any] = { + "posed_joints": posed_joints, + "foot_contacts": foot_contacts, + "lengths": lengths, + "constraints_lst": constraints_lst, + } + text_this = texts_parsed[0] if texts_parsed else "" + embs = _load_pair_embeddings(sample_dir) + if (text_this or "").strip() and embs is not None: + motion_emb, text_emb, gt_motion_emb = embs + metrics_in["motion_emb"] = motion_emb + metrics_in["text_emb"] = text_emb + if gt_motion_emb is not None: + metrics_in["gt_motion_emb"] = gt_motion_emb + motion_embs.append(motion_emb) + text_embs.append(text_emb) + + compute_metrics(metrics_list, metrics_in) + + per_sample_gen = _per_sample_metrics_from_saved(metrics_list, n) + raw_aggregated_gen = aggregate_metrics(metrics_list) + aggregated_gen = {} + tmr_metrics: dict[str, float] = {} + has_text = len(motion_embs) == n and len(text_embs) == n + for key, v in raw_aggregated_gen.items(): + val = _to_scalar(v) + if key.startswith("TMR/"): + if has_text: + tmr_metrics[key] = val + else: + aggregated_gen[key] = val + if "constraint_root2d_err" in raw_aggregated_gen: + aggregated_gen["constraint_root2d_err_p95"] = _to_p95(raw_aggregated_gen["constraint_root2d_err"]) + + tmr_per_sample: list[dict[str, Any]] = [] + if has_text and motion_embs and text_embs and len(motion_embs) == n and len(text_embs) == n: + motion_emb_stack = np.stack(motion_embs, axis=0) + text_emb_stack = np.stack(text_embs, axis=0) + tmr_per_sample = compute_tmr_per_sample_retrieval(motion_emb_stack, text_emb_stack, sample_ids, texts, top_k=5) + + # ----- Pass 2: GT (gt_motion.npz only, no embeddings) ----- + clear_metrics(metrics_list) + for sample_dir, rel_path in tqdm(group, desc=f"GT ({group_name})" if group_name else "GT", unit="motion"): + posed_joints, foot_contacts = _load_npz_motion(sample_dir / "gt_motion.npz", device, soma30_skel) + nframes = posed_joints.shape[0] + lengths = torch.tensor(nframes, dtype=torch.long, device=device) + constraints_path = sample_dir / "constraints.json" + constraints_lst = ( + load_constraints_lst(str(constraints_path), skeleton=skeleton) if constraints_path.is_file() else [] + ) + metrics_in = { + "posed_joints": posed_joints, + "foot_contacts": foot_contacts, + "lengths": lengths, + "constraints_lst": constraints_lst, + } + compute_metrics(metrics_list, metrics_in) + + per_sample_gt = _per_sample_metrics_from_saved(metrics_list, n) + raw_aggregated_gt = aggregate_metrics(metrics_list) + aggregated_gt = {} + for key, v in raw_aggregated_gt.items(): + if key.startswith("TMR/"): + continue + aggregated_gt[key] = _to_scalar(v) + if "constraint_root2d_err" in raw_aggregated_gt: + aggregated_gt["constraint_root2d_err_p95"] = _to_p95(raw_aggregated_gt["constraint_root2d_err"]) + + return ( + per_sample_gen, + per_sample_gt, + aggregated_gen, + aggregated_gt, + tmr_metrics, + tmr_per_sample, + ) + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + +def main(): + parser = argparse.ArgumentParser( + description="Recursively evaluate generated motions; write metrics.json per folder and .json per parent.", + ) + parser.add_argument( + "folder", + type=Path, + help="Root folder to search recursively for meta.json + motion.npz + gt_motion.npz", + ) + parser.add_argument("--device", default=None, help="cuda/cpu. Default: auto") + args = parser.parse_args() + + folder = args.folder.resolve() + if not folder.is_dir(): + raise SystemExit(f"Folder does not exist: {folder}") + + device = args.device or ("cuda" if torch.cuda.is_available() else "cpu") + + examples = discover_motion_folders(folder) + if not examples: + raise SystemExit(f"No directories with meta.json, motion.npz, and gt_motion.npz found under {folder}") + print(f"Discovered {len(examples)} motion folders.") + + first_posed = np.load(examples[0][0] / "motion.npz")["posed_joints"] + num_joints = first_posed.shape[-2] + + # SOMA models could generate 30-joint output; upscale to 77 for evaluation + soma30_skel: SOMASkeleton30 | None = None + if num_joints == 30: + soma30_skel = SOMASkeleton30().to(device) + _ = soma30_skel.somaskel77 # trigger lazy init + soma30_skel.somaskel77.to(device) + skeleton = soma30_skel.somaskel77 + print("Detected SOMA30 motions; will upscale to SOMA77 for evaluation.") + else: + skeleton = build_skeleton(num_joints).to(device) + + fps = DEFAULT_FPS + kwargs = {"skeleton": skeleton, "fps": fps} + metrics_list = [ + FootSkateFromHeight(**kwargs), + FootSkateFromContacts(**kwargs), + FootContactConsistency(**kwargs), + FootSkateRatio(**kwargs), + ContraintFollow(**kwargs), + TMR_EmbeddingMetric(**kwargs), + ] + + groups = group_by_parent(examples) + for group in tqdm(groups, desc="Evaluating folders"): + sample_dirs = [g[0] for g in group] + folder_for_group = sample_dirs[0].parent + folder_name = folder_for_group.name + + ( + per_sample_gen, + per_sample_gt, + aggregated_gen, + aggregated_gt, + tmr_metrics, + tmr_per_sample, + ) = _run_eval_on_group(group, skeleton, metrics_list, device, group_name=folder_name, soma30_skel=soma30_skel) + + texts = [] + for sample_dir, _ in group: + meta = load_json(sample_dir / "meta.json") + texts_parsed, _ = parse_prompts_from_meta(meta) + texts.append(texts_parsed[0] if texts_parsed else "") + + for i, (sample_dir, _) in enumerate(group): + metrics_path = sample_dir / "metrics.json" + out = { + "num_motions": 1, + "folder": str(sample_dir), + "per_motion_mean_gen": per_sample_gen[i] if i < len(per_sample_gen) else {}, + "per_motion_mean_gt": per_sample_gt[i] if i < len(per_sample_gt) else {}, + } + if i < len(tmr_per_sample): + out["tmr"] = { + "t2m_rank": tmr_per_sample[i]["rank"], + "text": texts[i] if i < len(texts) else "", + "top5_retrieved": tmr_per_sample[i]["top_k"], + } + _write_json(metrics_path, out) + + parent_json_path = folder_for_group.parent / f"{folder_name}.json" + full_metrics = { + "num_motions": len(group), + "folder": str(folder_for_group), + "per_motion_mean_gen": aggregated_gen, + "per_motion_mean_gt": aggregated_gt, + } + if tmr_metrics: + full_metrics["tmr"] = tmr_metrics + _write_json(parent_json_path, full_metrics) + + print(f"Wrote metrics.json in each of {len(examples)} folders and folder-level JSONs for {len(groups)} groups.") + + +if __name__ == "__main__": + main() diff --git a/benchmark/generate_eval.py b/benchmark/generate_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..894be374bf8f08d5c897cf06b5d8547f1b879de8 --- /dev/null +++ b/benchmark/generate_eval.py @@ -0,0 +1,382 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Step (2) of evaluation pipeline. + +This script recursively generates motions using Kimodo from a test suite folder tree. +""" + +import argparse +import shutil +from pathlib import Path +from typing import Any + +import numpy as np +import torch +from torch.utils.data import DataLoader, Dataset +from tqdm.auto import tqdm + +from kimodo.constraints import load_constraints_lst +from kimodo.meta import parse_prompts_from_meta +from kimodo.model import DEFAULT_MODEL, load_model +from kimodo.tools import load_json, seed_everything + + +def parse_args(): + parser = argparse.ArgumentParser(description="Recursively generate motions from a testsuite folder tree") + parser.add_argument( + "--benchmark", + type=str, + default="testsuite", + help="Root folder containing subfolders with meta.json (default: testsuite)", + ) + parser.add_argument( + "--output", + type=str, + default=None, + help="Output root; directory hierarchy is mirrored here. If omitted, motions are generated in-place inside the testsuite folder.", + ) + parser.add_argument( + "--batch_size", + type=int, + default=32, + help="Batch size for generating motions (default: 32)", + ) + parser.add_argument( + "--num_workers", + type=int, + default=4, + help="DataLoader workers for loading meta/constraints paths (default: 4)", + ) + parser.add_argument( + "--model", + type=str, + default=DEFAULT_MODEL, + help="Name of the model (e.g. Kimodo-SOMA-RP-v1.1, kimodo-soma-rp, or SOMA).", + ) + parser.add_argument( + "--diffusion_steps", + type=int, + default=100, + help="Number of diffusion steps (default: 100); overridden by meta.json if present", + ) + parser.add_argument( + "--postprocess", + action="store_true", + help="Apply motion post-processing to reduce foot skating", + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Regenerate outputs even if motion.npz already exists", + ) + parser.add_argument( + "--text_encoder_fp32", + action="store_true", + help="Uses fp32 for instantiating the text encoder (if API is not already running) rather than default bfloat16.", + ) + return parser.parse_args() + + +def discover_example_folders(root: Path) -> list[tuple[Path, Path]]: + """Discover leaf directories that contain meta.json. + + Returns list of (src_dir, rel_path). + """ + root = root.resolve() + if not root.is_dir(): + raise FileNotFoundError(f"Testsuite folder does not exist: {root}") + out: list[tuple[Path, Path]] = [] + for meta_path in root.rglob("meta.json"): + src_dir = meta_path.parent + rel = src_dir.relative_to(root) + out.append((src_dir, rel)) + return sorted(out, key=lambda x: str(x[1])) + + +def copy_source_files(src_dir: Path, out_dir: Path) -> None: + """Copy meta.json, constraints.json, and gt_motion.npz (if present) from src_dir to out_dir.""" + out_dir.mkdir(parents=True, exist_ok=True) + for name in ("meta.json", "constraints.json", "gt_motion.npz"): + src_file = src_dir / name + if src_file.is_file(): + shutil.copy2(src_file, out_dir / name) + + +class EvalExampleDataset(Dataset): + """Dataset of example folders: yields text, num_frame, constraints_path (and paths, meta). + No torch/skeleton in workers so num_workers > 0 is safe with CUDA. + """ + + def __init__( + self, + examples: list[tuple[Path, Path]], + testsuite_root: Path, + generated_root: Path, + fps: float, + ): + self.examples = examples + self.testsuite_root = testsuite_root + self.generated_root = generated_root + self.fps = fps + + def __len__(self) -> int: + return len(self.examples) + + def __getitem__(self, idx: int) -> dict[str, Any]: + src_dir, rel_path = self.examples[idx] + out_dir = self.generated_root / rel_path + meta_path = src_dir / "meta.json" + meta = load_json(str(meta_path)) + assert meta.get("num_samples", 1) == 1, "Expected num_samples to be absent or 1 in meta.json" + texts, durations_sec = parse_prompts_from_meta(meta) + assert len(texts) == 1, "Expected exactly one prompt (len(texts)==1) per example" + num_frames = [int(float(d) * self.fps) for d in durations_sec] + assert len(num_frames) == 1, "Expected exactly one duration per example" + constraints_path = src_dir / "constraints.json" + cpath = str(constraints_path) if constraints_path.is_file() else None + return { + "rel_path": rel_path, + "src_dir": str(src_dir), + "out_dir": str(out_dir), + "meta": meta, + "text": texts[0], + "num_frame": num_frames[0], + "constraints_path": cpath, + } + + +def collate_examples(batch: list[dict]) -> dict[str, Any]: + """Collate list of example dicts; keep list fields as lists (no stacking).""" + if not batch: + return {} + keys = batch[0].keys() + out: dict[str, Any] = {} + for k in keys: + vals = [b[k] for b in batch] + out[k] = vals + return out + + +def group_by_parent( + examples: list[tuple[Path, Path]], +) -> list[list[tuple[Path, Path]]]: + """Group (src_dir, rel_path) by parent directory of rel_path for folder-by-folder processing.""" + from itertools import groupby + + def parent_key(item: tuple[Path, Path]) -> Path: + rel = item[1] + return rel.parent if len(rel.parts) > 1 else Path(".") + + sorted_examples = sorted(examples, key=parent_key) + groups: list[list[tuple[Path, Path]]] = [] + for _key, group in groupby(sorted_examples, key=parent_key): + groups.append(list(group)) + return groups + + +def _slice_output_at(output: dict[str, Any], index: int) -> dict[str, Any]: + """Slice a (possibly nested) output dict at batch index for one sample.""" + out: dict[str, Any] = {} + for k, v in output.items(): + if isinstance(v, dict): + out[k] = _slice_output_at(v, index) + elif isinstance(v, np.ndarray) and v.ndim > 0: + out[k] = v[index] + else: + out[k] = v + return out + + +def _crop_output(output: dict[str, Any], num_frames: int) -> dict[str, Any]: + """Crop a single-sample output dict along the time dimension (axis 0).""" + out: dict[str, Any] = {} + for k, v in output.items(): + if isinstance(v, dict): + out[k] = _crop_output(v, num_frames) + elif isinstance(v, np.ndarray) and v.ndim >= 1: + out[k] = v[:num_frames] + else: + out[k] = v + return out + + +def main(): + device = "cuda:0" if torch.cuda.is_available() else "cpu" + print(f"Using device: {device}") + + args = parse_args() + testsuite_root = Path(args.benchmark).resolve() + if args.output is not None: + generated_root = Path(args.output).resolve() + else: + generated_root = testsuite_root + in_place = generated_root == testsuite_root + + examples = discover_example_folders(testsuite_root) + if not examples: + raise SystemExit(f"No folders with meta.json found under {testsuite_root}") + print(f"Discovered {len(examples)} example folders.") + + model, resolved_name = load_model( + args.model, + device=device, + default_family="Kimodo", + return_resolved_name=True, + text_encoder_fp32=args.text_encoder_fp32, + ) + # v1.1 models are meant to be used for benchmark evaluation + _deprecated_for_benchmark = { + "kimodo-soma-rp-v1": "Kimodo-SOMA-RP-v1 was not trained to be compatible with the benchmark evaluation.", + "kimodo-soma-seed-v1": "Kimodo-SOMA-SEED-v1 is not the latest model for benchmark evaluation.", + } + if resolved_name in _deprecated_for_benchmark: + import warnings + + warnings.warn( + f"Model '{args.model}' resolved to {resolved_name}: " + f"{_deprecated_for_benchmark[resolved_name]} Consider using v1.1.", + stacklevel=1, + ) + print(f"Generating with model: {resolved_name}") + fps = model.fps + default_diffusion_steps = args.diffusion_steps + + groups = group_by_parent(examples) + total_generated = 0 + total_skipped = 0 + + total_examples = len(examples) + for group in groups: + rel_path_0 = group[0][1] + if rel_path_0.parent != Path("."): + folder_label = str(rel_path_0.parent) + else: + # Direct children of testsuite root: show root name (e.g. inbetweening) + folder_label = testsuite_root.name + num_in_folder = len(group) + print(f"Generating folder: {folder_label} ({num_in_folder} motions)") + + dataset = EvalExampleDataset( + group, + testsuite_root, + generated_root, + fps=fps, + ) + loader = DataLoader( + dataset, + batch_size=args.batch_size, + shuffle=False, + num_workers=args.num_workers, + collate_fn=collate_examples, + ) + + folder_generated = 0 + folder_skipped = 0 + for batch_idx, batch in enumerate(loader): + rel_paths = batch["rel_path"] + src_dirs = batch["src_dir"] + out_dirs = batch["out_dir"] + metas = batch["meta"] + batch_texts = batch["text"] + batch_num_frames = batch["num_frame"] + constraints_paths = batch["constraints_path"] + + # Filter out samples that are already generated (unless --overwrite). + if args.overwrite: + selected_indices = list(range(len(rel_paths))) + else: + selected_indices = [] + for i, out_dir_str in enumerate(out_dirs): + motion_path = Path(out_dir_str) / "motion.npz" + if motion_path.is_file(): + folder_skipped += 1 + total_skipped += 1 + continue + selected_indices.append(i) + + if not selected_indices: + print( + f"\r Generated {folder_generated} / {num_in_folder} (skipped: {folder_skipped}) " + f"(total: {total_generated + total_skipped} / {total_examples})", + end="", + flush=True, + ) + continue + + rel_paths = [rel_paths[i] for i in selected_indices] + src_dirs = [src_dirs[i] for i in selected_indices] + out_dirs = [out_dirs[i] for i in selected_indices] + metas = [metas[i] for i in selected_indices] + batch_texts = [batch_texts[i] for i in selected_indices] + batch_num_frames = [batch_num_frames[i] for i in selected_indices] + constraints_paths = [constraints_paths[i] for i in selected_indices] + + # Load constraints in main process on model device (no torch in workers) + device_t = torch.device(device) + batch_constraints_lst = [ + load_constraints_lst(cpath, model.skeleton, device=device_t) if cpath else [] + for cpath in constraints_paths + ] + + if not in_place: + for i in range(len(rel_paths)): + copy_source_files(Path(src_dirs[i]), Path(out_dirs[i])) + + # Use first example's diffusion_steps and seed for the whole batch + diffusion_steps = metas[0].get("diffusion_steps", default_diffusion_steps) + seed = metas[0].get("seed", None) + if seed is not None: + seed_everything(seed) + else: + print("Warning: No seed found in meta.json, not seeding this batch.") + + # Single model call for the entire batch (count in bar title, bar clears when done) + bar_desc = ( + f" Generated {folder_generated} / {num_in_folder} " + f"(skipped: {folder_skipped}) (total: {total_generated + total_skipped} / {total_examples})" + ) + output = model( + batch_texts, + batch_num_frames, + constraint_lst=batch_constraints_lst, + num_denoising_steps=diffusion_steps, + multi_prompt=False, + post_processing=args.postprocess, + return_numpy=True, + progress_bar=lambda x: tqdm(x, leave=False, desc=bar_desc), + ) + + # Save each sample to its output dir + B = len(batch_texts) + for b in range(B): + out_dir = Path(out_dirs[b]) + sample_output = _slice_output_at(output, b) + sample_output = _crop_output(sample_output, batch_num_frames[b]) + motion_path = out_dir / "motion.npz" + np.savez(motion_path, **sample_output) + total_generated += 1 + folder_generated += 1 + + print( + f"\r Generated {folder_generated} / {num_in_folder} (skipped: {folder_skipped}) " + f"(total: {total_generated + total_skipped} / {total_examples})", + end="", + flush=True, + ) + + print() + print( + f" Finished folder {folder_label} ({num_in_folder} motions, " + f"generated: {folder_generated}, skipped: {folder_skipped})." + ) + + if in_place: + print(f"Generated {total_generated} motions in-place under {testsuite_root}.") + else: + print(f"Generated {total_generated} motions under {generated_root}.") + + +if __name__ == "__main__": + main() diff --git a/benchmark/parse_folder.py b/benchmark/parse_folder.py new file mode 100644 index 0000000000000000000000000000000000000000..6034247858062df6399a312a597d8bed82375049 --- /dev/null +++ b/benchmark/parse_folder.py @@ -0,0 +1,708 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Step (5) of evaluation pipeline. + +Validate testcase result JSONs and aggregate benchmark rows. + +Expected testsuite layout (aligned with evaluate_folder output): + + / + ├── / # e.g. content, repetition + │ ├── text2motion/ # text-following eval + │ │ ├── overview/ # or timeline_single, timeline_multi + │ │ │ └── .json + │ │ └── ... + │ └── / # constraints_withtext, constraints_notext + │ └── .../ # optional subdirs, e.g. root, fullbody + │ └── / + │ └── .json + +Samples are discovered via rglob('meta.json') with motion.npz and gt_motion.npz in the same dir. +Testcase dir = parent of a sample dir. Result file = testcase_dir.parent / f"{testcase_dir.name}.json". +""" + +from __future__ import annotations + +import argparse +import json +from collections import defaultdict +from pathlib import Path +from typing import Any + +SPLITS = ("content", "repetition") +TEXT_FOLLOWING_CATEGORIES = ("overview", "timeline_single", "timeline_multi") +CONSTRAINTS_CATEGORIES = ("constraints_withtext", "constraints_notext") +ROW_CATEGORIES = TEXT_FOLLOWING_CATEGORIES + CONSTRAINTS_CATEGORIES + + +def _discover_sample_dirs(root: Path) -> list[Path]: + sample_dirs: list[Path] = [] + for meta_path in root.rglob("meta.json"): + sample_dir = meta_path.parent + if (sample_dir / "motion.npz").is_file() and (sample_dir / "gt_motion.npz").is_file(): + sample_dirs.append(sample_dir) + return sorted(set(sample_dirs)) + + +def _discover_testcase_dirs(root: Path) -> list[Path]: + sample_dirs = _discover_sample_dirs(root) + return sorted({sample_dir.parent for sample_dir in sample_dirs}) + + +def _expected_result_path(testcase_dir: Path) -> Path: + return testcase_dir.parent / f"{testcase_dir.name}.json" + + +def _parse_testcase_key(root: Path, testcase_dir: Path) -> tuple[str, str]: + rel_parts = testcase_dir.relative_to(root).parts + if len(rel_parts) < 2: + raise ValueError(f"Unexpected testcase path shape: {testcase_dir} (relative: {'/'.join(rel_parts)})") + split = rel_parts[0] + if split not in SPLITS: + raise ValueError(f"Unknown split '{split}' for testcase {testcase_dir}") + if len(rel_parts) >= 3 and rel_parts[1] == "text2motion": + category = rel_parts[2] + if category not in TEXT_FOLLOWING_CATEGORIES: + raise ValueError(f"Unknown text-following category '{category}' for testcase {testcase_dir}") + else: + category = rel_parts[1] + if category not in CONSTRAINTS_CATEGORIES: + raise ValueError(f"Unknown category '{category}' for testcase {testcase_dir}") + return split, category + + +def _accumulate_weighted( + sum_acc: dict[str, float], + weight_acc: dict[str, float], + metric_dict: dict[str, Any], + weight: float, +) -> None: + for metric_name, value in metric_dict.items(): + if isinstance(value, (int, float)): + sum_acc[metric_name] = sum_acc.get(metric_name, 0.0) + float(value) * weight + weight_acc[metric_name] = weight_acc.get(metric_name, 0.0) + weight + + +def _to_averages( + weighted_sum: dict[str, float], weight: dict[str, float] +) -> dict[str, float]: + return { + k: v / weight[k] + for k, v in sorted(weighted_sum.items()) + if weight.get(k, 0.0) > 0 + } + + +def _load_result_row( + result_path: Path, +) -> tuple[float, dict[str, Any], dict[str, Any], dict[str, Any]]: + payload = json.loads(result_path.read_text(encoding="utf-8")) + num_motions = float(payload.get("num_motions", 1)) + per_motion_mean_gen = payload.get("per_motion_mean_gen") or payload.get("per_motion_mean", {}) + per_motion_mean_gt = payload.get("per_motion_mean_gt") or {} + tmr = payload.get("tmr") or {} + if not isinstance(per_motion_mean_gen, dict): + raise ValueError(f"'per_motion_mean_gen' / 'per_motion_mean' is not a dict in {result_path}") + if not isinstance(per_motion_mean_gt, dict): + raise ValueError(f"'per_motion_mean_gt' is not a dict in {result_path}") + if not isinstance(tmr, dict): + raise ValueError(f"'tmr' is not a dict in {result_path}") + return num_motions, per_motion_mean_gen, per_motion_mean_gt, tmr + + +# Display labels for table rows (paper-style). +TEXT_FOLLOWING_ROW_LABELS = { + "overview": "Overview", + "timeline_single": "Timeline single", + "timeline_multi": "Timeline multi", +} +CONSTRAINTS_ROW_LABELS = { + "constraints_withtext": "Constraints with text", + "constraints_notext": "Constraints without text", +} + +# Meters to cm for constraint position metrics. +M_TO_CM = 100.0 + + +def _table_value(val: float | None) -> float | str | None: + """Return value for JSON table; use None for missing (omit or serialize as null).""" + if val is None: + return None + if isinstance(val, (int, float)) and (val != val or val == float("inf")): # nan or inf + return None + return val + + +def _build_tables( + row_acc: dict[tuple[str, str], dict[str, Any]], +) -> dict[str, dict[str, list[dict[str, Any]]]]: + """Build text_following and constraints tables per split for paper-style output.""" + tables: dict[str, dict[str, list[dict[str, Any]]]] = {} + for split in SPLITS: + tables[split] = {"text_following": [], "constraints": []} + + # Text-following table: Overview, Timeline single, Timeline multi. + for category in TEXT_FOLLOWING_CATEGORIES: + acc = row_acc[(split, category)] + per_motion_gen = _to_averages(acc["per_motion_mean_weighted_sum"], acc["per_motion_mean_weight"]) + per_motion_gt = _to_averages(acc["per_motion_mean_gt_weighted_sum"], acc["per_motion_mean_gt_weight"]) + tmr_avg = _to_averages(acc["tmr_weighted_sum"], acc["tmr_weight"]) if acc["tmr_weight"] else {} + r03_gen = tmr_avg.get("TMR/t2m_R/R03") + r03_gt = tmr_avg.get("TMR/t2m_gt_R/R03") + fid_gen_text = tmr_avg.get("TMR/FID/gen_text") + fid_gt_text = tmr_avg.get("TMR/FID/gt_text") + fid_gen_gt = tmr_avg.get("TMR/FID/gen_gt") + # Skate is velocity in m/s; convert to cm/s for display. + skate_gen = per_motion_gen.get("foot_skate_from_pred_contacts") + skate_gt = per_motion_gt.get("foot_skate_from_pred_contacts") + contact_gen = per_motion_gen.get("foot_contact_consistency") + contact_gt = per_motion_gt.get("foot_contact_consistency") + row_label = TEXT_FOLLOWING_ROW_LABELS[category] + tables[split]["text_following"].append( + { + "row": row_label, + "R@3 (gen)": _table_value(r03_gen), + "R@3 (GT)": _table_value(r03_gt), + "FID gen-text": _table_value(fid_gen_text), + "FID GT-text": _table_value(fid_gt_text), + "FID gen-GT": _table_value(fid_gen_gt), + "Skate (gen, cm/s)": _table_value(skate_gen * 100.0 if skate_gen is not None else None), + "Skate (GT, cm/s)": _table_value(skate_gt * 100.0 if skate_gt is not None else None), + "Contact (gen)": _table_value(contact_gen), + "Contact (GT)": _table_value(contact_gt), + } + ) + + # Constraints table: Constraints with text, Constraints without text. + for category in CONSTRAINTS_CATEGORIES: + acc = row_acc[(split, category)] + per_motion_gen = _to_averages(acc["per_motion_mean_weighted_sum"], acc["per_motion_mean_weight"]) + per_motion_gt = _to_averages(acc["per_motion_mean_gt_weighted_sum"], acc["per_motion_mean_gt_weight"]) + row_label = CONSTRAINTS_ROW_LABELS[category] + row_dict: dict[str, Any] = { + "row": row_label, + "Full-Body Pos (gen, cm)": _table_value( + per_motion_gen.get("constraint_fullbody_keyframe") * M_TO_CM + if per_motion_gen.get("constraint_fullbody_keyframe") is not None + else None + ), + "Full-Body Pos (GT, cm)": _table_value( + per_motion_gt.get("constraint_fullbody_keyframe") * M_TO_CM + if per_motion_gt.get("constraint_fullbody_keyframe") is not None + else None + ), + "End-Effector Pos (gen, cm)": _table_value( + per_motion_gen.get("constraint_end_effector") * M_TO_CM + if per_motion_gen.get("constraint_end_effector") is not None + else None + ), + "End-Effector Pos (GT, cm)": _table_value( + per_motion_gt.get("constraint_end_effector") * M_TO_CM + if per_motion_gt.get("constraint_end_effector") is not None + else None + ), + "End-Effector Rot (deg)": None, # Not implemented in metrics. + "2D Root Pos (gen, cm)": _table_value( + per_motion_gen.get("constraint_root2d_err") * M_TO_CM + if per_motion_gen.get("constraint_root2d_err") is not None + else None + ), + "2D Root Pos (GT, cm)": _table_value( + per_motion_gt.get("constraint_root2d_err") * M_TO_CM + if per_motion_gt.get("constraint_root2d_err") is not None + else None + ), + "2D Pelvis Pos@95% (gen, cm)": _table_value( + per_motion_gen.get("constraint_root2d_err_p95") * M_TO_CM + if per_motion_gen.get("constraint_root2d_err_p95") is not None + else None + ), + "2D Pelvis Pos@95% (GT, cm)": _table_value( + per_motion_gt.get("constraint_root2d_err_p95") * M_TO_CM + if per_motion_gt.get("constraint_root2d_err_p95") is not None + else None + ), + } + tables[split]["constraints"].append(row_dict) + + return tables + + +def _fmt_md(val: float | None, decimals: int) -> str: + """Format a numeric value for a markdown cell, or '-' for None/NaN.""" + if val is None: + return "-" + if isinstance(val, float) and (val != val or val == float("inf")): + return "-" + return f"{val:.{decimals}f}" + + +def _print_tf_formatted_md( + splits_data: list[tuple[str, list[dict[str, Any]]]], + title: str, +) -> None: + """Print text-following table in markdown, mirroring the terminal layout.""" + groups = ["Overview", "Timeline single", "Timeline multi"] + specs: list[tuple[str, int]] = [ + ("R@3\u2191", 2), + ("FID\u2193", 3), + ("Skate\u2193", 3), + ("Contact\u2191", 3), + ] + gt_keys = ["R@3 (GT)", None, "Skate (GT, cm/s)", "Contact (GT)"] + gen_keys = ["R@3 (gen)", "FID gen-GT", "Skate (gen, cm/s)", "Contact (gen)"] + gt_defaults: list[float | None] = [None, 0.0, None, None] + + headers = [""] + for g in groups: + for hdr, _ in specs: + headers.append(f"{g} {hdr}") + + print(f"\n### {title}\n") + print("| " + " | ".join(headers) + " |") + print("| " + " | ".join("---" for _ in headers) + " |") + + for split_label, rows in splits_data: + for row_type, keys, defaults in [ + ("Ground Truth", gt_keys, gt_defaults), + ("Method", gen_keys, [None] * len(specs)), + ]: + cells = [f"**{split_label}** {row_type}"] + for row in rows: + for j, (_, dec) in enumerate(specs): + key = keys[j] + val = defaults[j] if key is None else row.get(key) + cells.append(_fmt_md(val, dec)) + print("| " + " | ".join(cells) + " |") + + print() + + +def _print_c_formatted_md( + splits_data: list[tuple[str, list[dict[str, Any]]]], + title: str, +) -> None: + """Print constraints table in markdown, mirroring the terminal layout.""" + groups = ["With text", "Without text"] + specs: list[tuple[str, int]] = [ + ("FB Pos\u2193", 3), + ("EE Pos\u2193", 3), + ("EE Rot\u2193", 3), + ("2D Root\u2193", 3), + ("Pelvis@95%", 2), + ] + gt_keys = [ + "Full-Body Pos (GT, cm)", + "End-Effector Pos (GT, cm)", + "End-Effector Rot (deg)", + "2D Root Pos (GT, cm)", + "2D Pelvis Pos@95% (GT, cm)", + ] + gen_keys = [ + "Full-Body Pos (gen, cm)", + "End-Effector Pos (gen, cm)", + "End-Effector Rot (deg)", + "2D Root Pos (gen, cm)", + "2D Pelvis Pos@95% (gen, cm)", + ] + + headers = [""] + for g in groups: + for hdr, _ in specs: + headers.append(f"{g} {hdr}") + + print(f"\n### {title}\n") + print("| " + " | ".join(headers) + " |") + print("| " + " | ".join("---" for _ in headers) + " |") + + for split_label, rows in splits_data: + for row_type, keys in [("Ground Truth", gt_keys), ("Method", gen_keys)]: + cells = [f"**{split_label}** {row_type}"] + for row in rows: + for j, (_, dec) in enumerate(specs): + cells.append(_fmt_md(row.get(keys[j]), dec)) + print("| " + " | ".join(cells) + " |") + + print() + + +def _print_formatted_gt_method_md( + tables: dict[str, dict[str, list[dict[str, Any]]]], +) -> None: + """Print combined tables in markdown format, mirroring the terminal layout.""" + tf_splits: list[tuple[str, list[dict[str, Any]]]] = [] + c_splits: list[tuple[str, list[dict[str, Any]]]] = [] + for split in SPLITS: + split_tables = tables.get(split, {}) + tf_rows = split_tables.get("text_following", []) + c_rows = split_tables.get("constraints", []) + if tf_rows and len(tf_rows) == 3: + tf_splits.append((split.capitalize(), tf_rows)) + if c_rows and len(c_rows) == 2: + c_splits.append((split.capitalize(), c_rows)) + + if tf_splits: + _print_tf_formatted_md(tf_splits, "Text-Following Evaluation") + if c_splits: + _print_c_formatted_md(c_splits, "Constrained Evaluation") + + +def _fmt(val: float | None, decimals: int, width: int) -> str: + """Format a numeric value right-aligned to *width*, or '-' for None.""" + if val is None: + return f"{'-':>{width}}" + return f"{val:>{width}.{decimals}f}" + + +def _print_grouped_rows( + label: str, + rows: list[dict[str, Any]], + specs: list[tuple[str, int, int]], + keys: list[str], + mw: int, + sep: str, +) -> None: + """Print one data row across all column groups.""" + parts = [f"{label:<{mw}}"] + for i, row in enumerate(rows): + if i: + parts.append(sep) + for j, (_, dec, w) in enumerate(specs): + parts.append(_fmt(row.get(keys[j]), dec, w)) + print("".join(parts)) + + +def _print_tf_formatted( + splits_data: list[tuple[str, list[dict[str, Any]]]], + title: str, +) -> None: + """Print text-following table with Overview / Timeline single / Timeline multi groups. + + *splits_data* is a list of ``(split_label, category_rows)`` tuples so + that content and repetition splits appear as separate row-pairs inside + one table. + """ + groups = ["Overview", "Timeline single", "Timeline multi"] + specs: list[tuple[str, int, int]] = [ + ("R@3\u2191", 2, 7), + ("FID\u2193", 3, 7), + ("Skate\u2193", 3, 9), + ("Contact\u2191", 3, 10), + ] + gt_keys = ["R@3 (GT)", None, "Skate (GT, cm/s)", "Contact (GT)"] + gen_keys = ["R@3 (gen)", "FID gen-GT", "Skate (gen, cm/s)", "Contact (gen)"] + gt_defaults: list[float | None] = [None, 0.0, None, None] + + mw = 16 + gw = sum(s[2] for s in specs) + sep = " | " + total_w = mw + len(groups) * gw + (len(groups) - 1) * len(sep) + + print(f"\n{title:^{total_w}}") + print("=" * total_w) + + parts: list[str] = [" " * mw] + for i, g in enumerate(groups): + if i: + parts.append(sep) + parts.append(g.center(gw)) + print("".join(parts)) + + parts = [f"{'':<{mw}}"] + for i in range(len(groups)): + if i: + parts.append(sep) + for hdr, _, w in specs: + parts.append(f"{hdr:>{w}}") + print("".join(parts)) + + parts = ["\u2500" * mw] + for i in range(len(groups)): + if i: + parts.append("\u2500\u253c\u2500") + parts.append("\u2500" * gw) + print("".join(parts)) + + for si, (split_label, rows) in enumerate(splits_data): + tag = f"\u2500\u2500 {split_label} " + print(tag + "\u2500" * (total_w - len(tag))) + + parts = [f"{'Ground Truth':<{mw}}"] + for i, row in enumerate(rows): + if i: + parts.append(sep) + for j, (_, dec, w) in enumerate(specs): + key = gt_keys[j] + val = gt_defaults[j] if key is None else row.get(key) + parts.append(_fmt(val, dec, w)) + print("".join(parts)) + + _print_grouped_rows("Method", rows, specs, gen_keys, mw, sep) + + print() + + +def _print_c_formatted( + splits_data: list[tuple[str, list[dict[str, Any]]]], + title: str, +) -> None: + """Print constraints table with With text / Without text groups. + + *splits_data* is a list of ``(split_label, category_rows)`` tuples. + """ + groups = ["With text", "Without text"] + specs: list[tuple[str, int, int]] = [ + ("FB Pos\u2193", 3, 10), + ("EE Pos\u2193", 3, 10), + ("EE Rot\u2193", 3, 10), + ("2D Root\u2193", 3, 11), + ("Pelvis@95%", 2, 12), + ] + gt_keys = [ + "Full-Body Pos (GT, cm)", + "End-Effector Pos (GT, cm)", + "End-Effector Rot (deg)", + "2D Root Pos (GT, cm)", + "2D Pelvis Pos@95% (GT, cm)", + ] + gen_keys = [ + "Full-Body Pos (gen, cm)", + "End-Effector Pos (gen, cm)", + "End-Effector Rot (deg)", + "2D Root Pos (gen, cm)", + "2D Pelvis Pos@95% (gen, cm)", + ] + + mw = 16 + gw = sum(s[2] for s in specs) + sep = " | " + total_w = mw + len(groups) * gw + (len(groups) - 1) * len(sep) + + print(f"\n{title:^{total_w}}") + print("=" * total_w) + + parts: list[str] = [" " * mw] + for i, g in enumerate(groups): + if i: + parts.append(sep) + parts.append(g.center(gw)) + print("".join(parts)) + + parts = [f"{'':<{mw}}"] + for i in range(len(groups)): + if i: + parts.append(sep) + for hdr, _, w in specs: + parts.append(f"{hdr:>{w}}") + print("".join(parts)) + + parts = ["\u2500" * mw] + for i in range(len(groups)): + if i: + parts.append("\u2500\u253c\u2500") + parts.append("\u2500" * gw) + print("".join(parts)) + + for si, (split_label, rows) in enumerate(splits_data): + tag = f"\u2500\u2500 {split_label} " + print(tag + "\u2500" * (total_w - len(tag))) + + _print_grouped_rows("Ground Truth", rows, specs, gt_keys, mw, sep) + _print_grouped_rows("Method", rows, specs, gen_keys, mw, sep) + + print() + + +def _print_formatted_gt_method( + tables: dict[str, dict[str, list[dict[str, Any]]]], +) -> None: + """Print combined tables with column groups separated by vertical bars. + + Content and repetition splits are shown as separate row-pairs inside one text-following table + and one constraints table. + """ + tf_splits: list[tuple[str, list[dict[str, Any]]]] = [] + c_splits: list[tuple[str, list[dict[str, Any]]]] = [] + for split in SPLITS: + split_tables = tables.get(split, {}) + tf_rows = split_tables.get("text_following", []) + c_rows = split_tables.get("constraints", []) + if tf_rows and len(tf_rows) == 3: + tf_splits.append((split.capitalize(), tf_rows)) + if c_rows and len(c_rows) == 2: + c_splits.append((split.capitalize(), c_rows)) + + if tf_splits: + _print_tf_formatted(tf_splits, "Text-Following Evaluation") + if c_splits: + _print_c_formatted(c_splits, "Constrained Evaluation") + + +def _build_summary(root: Path) -> dict[str, Any]: + testcase_dirs = _discover_testcase_dirs(root) + if not testcase_dirs: + raise SystemExit( + f"No testcase folders found under {root} (expected folders containing meta.json + motion.npz + gt_motion.npz samples)." + ) + + missing_results: list[Path] = [] + for testcase_dir in testcase_dirs: + result_path = _expected_result_path(testcase_dir) + if not result_path.is_file(): + missing_results.append(result_path) + + if missing_results: + missing_text = "\n".join(str(path) for path in missing_results) + raise SystemExit(f"Missing {len(missing_results)} testcase result JSON files:\n{missing_text}") + + row_acc: dict[tuple[str, str], dict[str, Any]] = {} + for split in SPLITS: + for category in ROW_CATEGORIES: + row_acc[(split, category)] = { + "num_testcases": 0, + "num_motions": 0.0, + "per_motion_mean_weighted_sum": {}, + "per_motion_mean_weight": {}, + "per_motion_mean_gt_weighted_sum": {}, + "per_motion_mean_gt_weight": {}, + "tmr_weighted_sum": {}, + "tmr_weight": {}, + } + + for testcase_dir in testcase_dirs: + split, category = _parse_testcase_key(root, testcase_dir) + result_path = _expected_result_path(testcase_dir) + num_motions, per_motion_mean_gen, per_motion_mean_gt, tmr = _load_result_row(result_path) + + acc = row_acc[(split, category)] + acc["num_testcases"] += 1 + acc["num_motions"] += num_motions + _accumulate_weighted( + acc["per_motion_mean_weighted_sum"], + acc["per_motion_mean_weight"], + per_motion_mean_gen, + num_motions, + ) + if per_motion_mean_gt: + _accumulate_weighted( + acc["per_motion_mean_gt_weighted_sum"], + acc["per_motion_mean_gt_weight"], + per_motion_mean_gt, + num_motions, + ) + if tmr: + _accumulate_weighted( + acc["tmr_weighted_sum"], + acc["tmr_weight"], + tmr, + num_motions, + ) + + rows: list[dict[str, Any]] = [] + for split in SPLITS: + for category in ROW_CATEGORIES: + acc = row_acc[(split, category)] + tmr_avg = _to_averages(acc["tmr_weighted_sum"], acc["tmr_weight"]) if acc["tmr_weight"] else {} + per_motion_gt_avg = _to_averages(acc["per_motion_mean_gt_weighted_sum"], acc["per_motion_mean_gt_weight"]) + row_dict: dict[str, Any] = { + "split": split, + "category": category, + "num_testcases": acc["num_testcases"], + "num_motions": int(acc["num_motions"]), + "per_motion_mean": _to_averages(acc["per_motion_mean_weighted_sum"], acc["per_motion_mean_weight"]), + "tmr": tmr_avg, + } + if per_motion_gt_avg: + row_dict["per_motion_mean_gt"] = per_motion_gt_avg + rows.append(row_dict) + + # Combined constraints row for this split. + withtext = row_acc[(split, "constraints_withtext")] + notext = row_acc[(split, "constraints_notext")] + + combined_per_motion = defaultdict(float) + combined_per_motion_weight = defaultdict(float) + combined_per_motion_gt = defaultdict(float) + combined_per_motion_gt_weight = defaultdict(float) + combined_tmr = defaultdict(float) + combined_tmr_weight = defaultdict(float) + for sum_key, weight_key, sum_acc, weight_acc in ( + ("per_motion_mean_weighted_sum", "per_motion_mean_weight", combined_per_motion, combined_per_motion_weight), + ("per_motion_mean_gt_weighted_sum", "per_motion_mean_gt_weight", combined_per_motion_gt, combined_per_motion_gt_weight), + ("tmr_weighted_sum", "tmr_weight", combined_tmr, combined_tmr_weight), + ): + for src in (withtext, notext): + for k, v in src[sum_key].items(): + sum_acc[k] += v + for k, w in src[weight_key].items(): + weight_acc[k] += w + + combined_tmr_avg = _to_averages(dict(combined_tmr), dict(combined_tmr_weight)) if combined_tmr_weight else {} + combined_gt_avg = _to_averages(dict(combined_per_motion_gt), dict(combined_per_motion_gt_weight)) + combined_row: dict[str, Any] = { + "split": split, + "category": "constraints", + "num_testcases": withtext["num_testcases"] + notext["num_testcases"], + "num_motions": int(withtext["num_motions"] + notext["num_motions"]), + "per_motion_mean": _to_averages(dict(combined_per_motion), dict(combined_per_motion_weight)), + "tmr": combined_tmr_avg, + } + if combined_gt_avg: + combined_row["per_motion_mean_gt"] = combined_gt_avg + rows.append(combined_row) + + tables = _build_tables(row_acc) + return { + "folder": str(root), + "num_testcases": len(testcase_dirs), + "rows": rows, + "tables": tables, + } + + +def main() -> None: + parser = argparse.ArgumentParser( + description=("Validate testcase XXX.json result files and aggregate averages by split/category.") + ) + parser.add_argument( + "folder", + type=Path, + help="Testsuite root folder (contains content/ and repetition/).", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Optional output JSON path. Default: /summary_rows.json", + ) + parser.add_argument( + "--format", + choices=["terminal", "md"], + default="terminal", + dest="table_format", + help="Table output format: 'terminal' (default) for fixed-width tables, 'md' for markdown.", + ) + args = parser.parse_args() + + folder = args.folder.resolve() + if not folder.is_dir(): + raise SystemExit(f"Folder does not exist: {folder}") + + summary = _build_summary(folder) + + out_path = args.output.resolve() if args.output else folder / "summary_rows.json" + out_path.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(f"Wrote aggregated summary: {out_path}") + print(f"Rows: {len(summary['rows'])}, testcases: {summary['num_testcases']}") + if args.table_format == "md": + _print_formatted_gt_method_md(summary["tables"]) + else: + _print_formatted_gt_method(summary["tables"]) + + +if __name__ == "__main__": + main() diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f34b1ed3dd27ee167bfb103671a39247227ed00c --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,92 @@ +services: + text-encoder: + build: + context: . + dockerfile: Dockerfile + image: kimodo:1.0 + container_name: text-encoder + working_dir: /workspace + command: python -m kimodo.scripts.run_text_encoder_server + volumes: + - ./:/workspace + # Cache HF downloads in host "system-wide" Hugging Face cache. + - ${HOME}/.cache/huggingface:/workspace/.cache/huggingface + # Mount the host HF auth token at the standard cache location in-container. + - ${HOME}/.cache/huggingface/token:/workspace/.cache/huggingface/token:ro + # expose to your host browser + ports: + - "9550:9550" + environment: + # Make Gradio reachable from other containers + # - GRADIO_SERVER_NAME=0.0.0.0 + # - GRADIO_SERVER_PORT=9550 + - HF_HOME=/workspace/.cache/huggingface + # Host user mapping (for non-root ownership + proper shell prompt) + - HOST_USER=${USER:-user} + + # GPU + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + + shm_size: "16gb" + ipc: host + + # Wait until Gradio responds on HTTP + healthcheck: + test: + ["CMD", "bash", "-lc", "curl -fsS http://localhost:9550/ > /dev/null"] + interval: 3s + timeout: 2s + retries: 40 + + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + + demo: + build: + context: . + dockerfile: Dockerfile + image: kimodo:1.0 + container_name: demo + working_dir: /workspace + command: python -m kimodo.demo + volumes: + - ./:/workspace + - ${HOME}/.cache/huggingface:/workspace/.cache/huggingface + - ${HOME}/.cache/huggingface/token:/workspace/.cache/huggingface/token:ro + # Explicit checkpoint mount (avoids surprises if the repo bind mount isn't what you expect). + - ./checkpoints:/workspace/checkpoints:ro + ports: + - "${SERVER_PORT:-7860}:${SERVER_PORT:-7860}" + environment: + # Point the model at the text-encoder service. + - TEXT_ENCODER_URL=http://text-encoder:9550/ + # Make checkpoint paths robust (Hydra config reads this). + - SERVER_PORT=${SERVER_PORT:-7860} + - HF_HOME=/workspace/.cache/huggingface + # Host user mapping (for non-root ownership + proper shell prompt) + - HOST_USER=${USER:-user} + + # GPU + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + + shm_size: "16gb" + ipc: host + + depends_on: + text-encoder: + condition: service_healthy + + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] diff --git a/docker_requirements.in b/docker_requirements.in new file mode 100644 index 0000000000000000000000000000000000000000..3d8dd88324885e47bc664263caf055a2a456c8d9 --- /dev/null +++ b/docker_requirements.in @@ -0,0 +1,49 @@ +# +# Human-maintained direct dependencies (top-level). +# Use `uv` to compile this into a fully pinned `requirements.txt` lockfile. +# +# IMPORTANT: +# - We intentionally do NOT list `torch` here because the Docker image base +# (`nvcr.io/nvidia/pytorch`) already provides it. Installing torch via pip +# during image build is slow and can lead to ABI/CUDA mismatches. +# - If you are NOT using Docker, install an appropriate PyTorch build separately. +# + +# Config / wiring +hydra-core>=1.3 +omegaconf>=2.3 + +# Core numerics +numpy>=1.23,<2 +scipy>=1.10,<2 + +# Model / embeddings +# NOTE: `kimodo/model/llm2vec` is has only been tested with transformers==5.1.0 +transformers==5.1.0 +urllib3>=2.6.3 +boto3 +peft>=0.12 +einops>=0.7 + +# Misc +tqdm>=4.0 +packaging>=21.0 +pydantic>=2.0 + +# UI / client +filelock>=3.20.3 +gradio>=6.8.0 +gradio_client>=1.0 + +# Visualization +trimesh>=3.21.7 +scenepic>=1.1.0 +pillow>=9.0 +av>=16.1.0 + +py-soma-x @ git+https://github.com/NVlabs/SOMA-X.git + +# Local packages (editable installs for viser and kimodo; MotionCorrection non-editable) +./MotionCorrection +-e . +-e ./kimodo-viser diff --git a/docker_requirements.txt b/docker_requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..77a677fa18c6f1d239f93248187eac1735c2760a --- /dev/null +++ b/docker_requirements.txt @@ -0,0 +1,377 @@ +# This file was autogenerated by uv via the following command: +# NOTE: `torch` (and its CUDA wheels) are intentionally omitted from this lockfile. +# The Docker base image (nvcr.io/nvidia/pytorch) already provides a tested PyTorch build. +# +# uv pip compile docker_requirements.in -o docker_requirements.txt --python-version 3.10 --python-platform x86_64-manylinux2014 +-e . + # via -r docker_requirements.in +-e ./kimodo-viser + # via -r docker_requirements.in +py-soma-x @ git+https://github.com/NVlabs/SOMA-X.git + # via -r docker_requirements.in +accelerate==1.13.0 + # via peft +aiofiles==24.1.0 + # via gradio +annotated-doc==0.0.4 + # via + # fastapi + # typer +annotated-types==0.7.0 + # via pydantic +antlr4-python3-runtime==4.9.3 + # via + # hydra-core + # omegaconf +anyio==4.12.1 + # via + # gradio + # httpx + # starlette +attrs==25.4.0 + # via + # jsonschema + # referencing +av==16.1.0 + # via + # -r docker_requirements.in + # kimodo +boto3==1.42.66 + # via + # -r docker_requirements.in + # kimodo +botocore==1.42.66 + # via + # boto3 + # s3transfer +brotli==1.2.0 + # via gradio +certifi==2026.2.25 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.5 + # via + # requests + # trimesh +click==8.3.1 + # via + # typer + # uvicorn +colorlog==6.10.1 + # via trimesh +einops==0.8.2 + # via + # -r docker_requirements.in + # kimodo +embreex==2.17.7.post7 + # via trimesh +exceptiongroup==1.3.1 + # via anyio +fastapi==0.135.1 + # via gradio +ffmpy==1.0.0 + # via gradio +filelock==3.25.2 + # via + # -r docker_requirements.in + # huggingface-hub + # kimodo + # torch +fsspec==2026.2.0 + # via + # gradio-client + # huggingface-hub + # torch +gradio==6.9.0 + # via + # -r docker_requirements.in + # kimodo +gradio-client==2.3.0 + # via + # -r docker_requirements.in + # gradio + # kimodo +groovy==0.1.2 + # via gradio +h11==0.16.0 + # via + # httpcore + # uvicorn +hf-xet==1.4.0 + # via huggingface-hub +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via + # gradio + # gradio-client + # huggingface-hub + # safehttpx + # trimesh +huggingface-hub==1.6.0 + # via + # accelerate + # gradio + # gradio-client + # peft + # tokenizers + # transformers +hydra-core==1.3.2 + # via + # -r docker_requirements.in + # kimodo +idna==3.11 + # via + # anyio + # httpx + # requests +imageio==2.37.3 + # via viser +jinja2==3.1.6 + # via + # gradio + # torch +jmespath==1.1.0 + # via + # boto3 + # botocore +jsonschema==4.26.0 + # via trimesh +jsonschema-specifications==2025.9.1 + # via jsonschema +lxml==6.0.2 + # via + # trimesh + # yourdfpy +manifold3d==3.4.0 + # via trimesh +mapbox-earcut==2.0.0 + # via trimesh +markdown-it-py==4.0.0 + # via rich +markupsafe==3.0.3 + # via + # gradio + # jinja2 +mdurl==0.1.2 + # via markdown-it-py +./MotionCorrection + # via -r docker_requirements.in +msgspec==0.20.0 + # via viser +nodeenv==1.10.0 + # via viser +numpy==1.26.4 + # via + # -r docker_requirements.in + # accelerate + # embreex + # gradio + # imageio + # kimodo + # manifold3d + # mapbox-earcut + # motion-correction + # pandas + # peft + # pycollada + # scenepic + # scipy + # shapely + # transformers + # trimesh + # vhacdx + # viser + # yourdfpy +omegaconf==2.3.0 + # via + # -r docker_requirements.in + # hydra-core + # kimodo +orjson==3.11.7 + # via gradio +packaging==26.0 + # via + # -r docker_requirements.in + # accelerate + # gradio + # gradio-client + # huggingface-hub + # hydra-core + # kimodo + # peft + # transformers +pandas==2.3.3 + # via gradio +peft==0.18.1 + # via + # -r docker_requirements.in + # kimodo +pillow==12.1.1 + # via + # -r docker_requirements.in + # gradio + # imageio + # kimodo + # scenepic + # trimesh +psutil==7.2.2 + # via + # accelerate + # peft +pycollada==0.9.3 + # via trimesh +pydantic==2.12.5 + # via + # -r docker_requirements.in + # fastapi + # gradio + # kimodo +pydantic-core==2.41.5 + # via pydantic +pydub==0.25.1 + # via gradio +pygments==2.19.2 + # via rich +python-dateutil==2.9.0.post0 + # via + # botocore + # pandas + # pycollada +python-multipart==0.0.22 + # via gradio +pytz==2026.1.post1 + # via + # gradio + # pandas +pyyaml==6.0.3 + # via + # accelerate + # gradio + # huggingface-hub + # omegaconf + # peft + # transformers +referencing==0.37.0 + # via + # jsonschema + # jsonschema-specifications +regex==2026.2.28 + # via transformers +requests==2.32.5 + # via viser +rich==14.3.3 + # via + # typer + # viser +rpds-py==0.30.0 + # via + # jsonschema + # referencing +rtree==1.4.1 + # via trimesh +s3transfer==0.16.0 + # via boto3 +safehttpx==0.1.7 + # via gradio +safetensors==0.7.0 + # via + # accelerate + # peft + # transformers +scenepic==1.1.2 + # via + # -r docker_requirements.in + # kimodo +scipy==1.15.3 + # via + # -r docker_requirements.in + # kimodo + # scenepic + # trimesh +semantic-version==2.10.0 + # via gradio +shapely==2.1.2 + # via trimesh +shellingham==1.5.4 + # via typer +six==1.17.0 + # via + # python-dateutil + # yourdfpy +starlette==0.52.1 + # via + # fastapi + # gradio +svg-path==7.0 + # via trimesh +tokenizers==0.22.2 + # via transformers +tomlkit==0.13.3 + # via gradio +tqdm==4.67.3 + # via + # -r docker_requirements.in + # huggingface-hub + # kimodo + # peft + # transformers + # viser +transformers==5.1.0 + # via + # -r docker_requirements.in + # kimodo + # peft +trimesh==4.11.3 + # via + # -r docker_requirements.in + # kimodo + # viser + # yourdfpy +typer==0.24.1 + # via + # gradio + # huggingface-hub + # typer-slim +typer-slim==0.24.0 + # via transformers +typing-extensions==4.15.0 + # via + # anyio + # exceptiongroup + # fastapi + # gradio + # gradio-client + # huggingface-hub + # pydantic + # pydantic-core + # referencing + # starlette + # torch + # typing-inspection + # uvicorn + # viser +typing-inspection==0.4.2 + # via + # fastapi + # pydantic +tzdata==2025.3 + # via pandas +urllib3==2.6.3 + # via + # -r docker_requirements.in + # botocore + # kimodo + # requests +uvicorn==0.41.0 + # via gradio +vhacdx==0.0.10 + # via trimesh +websockets==15.0.1 + # via viser +xxhash==3.6.0 + # via trimesh +yourdfpy==0.0.60 + # via viser diff --git a/docs/.gitattributes b/docs/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..f5f7069a4309c21c065d50f9ffc610e9806000b2 --- /dev/null +++ b/docs/.gitattributes @@ -0,0 +1 @@ +source/_static/quick_tour.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f07f60c328bbad4d563776af62f62fbc0e7cf6d8 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,17 @@ +# Minimal makefile for Sphinx documentation +# + +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +.PHONY: help Makefile apidoc + +# Catch-all target: route all unknown targets to Sphinx +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) + +apidoc: + @$(SPHINXBUILD) -b html -q "$(SOURCEDIR)" "$(BUILDDIR)" >/dev/null 2>&1 || true + @sphinx-apidoc -o "$(SOURCEDIR)/api_reference/_generated" -t "$(SOURCEDIR)/_templates/apidoc" ../kimodo ../kimodo/**/tests* ../kimodo/**/test* -f diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f21230a384c18d11b0e7a9372584bfee8eb5626d --- /dev/null +++ b/docs/README.md @@ -0,0 +1,31 @@ +# Documentation + +## Local build + +Install doc dependencies: + +```bash +pip install -r docs/requirements.txt +``` + +Build HTML: + +```bash +cd docs +make html +``` + +Open the output at `docs/build/html/index.html`. + +## API reference generation + +Generate API stubs from the Python packages: + +```bash +cd docs +make apidoc +make html +``` + +Note: generated stubs are written to `docs/source/api_reference/_generated` and are not +included in the default navigation. Add them to a toctree if you want to expose them. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000000000000000000000000000000000000..66a55a9e8abeedd53f52906be7c00e100d199370 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,19 @@ +@ECHO OFF + +pushd %~dp0 + +set SPHINXOPTS= +set SPHINXBUILD=sphinx-build +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4d735dd9305f19eeb5bc7153d001fc8f570e00c0 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +sphinx>=7.0,<9.0 +nvidia-sphinx-theme +sphinx-copybutton +myst-parser +sphinx-design diff --git a/docs/source/_static/arch.png b/docs/source/_static/arch.png new file mode 100644 index 0000000000000000000000000000000000000000..9288f9edd8a386bf0803df990ab36f84e715abe7 --- /dev/null +++ b/docs/source/_static/arch.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c349298e5381fab632375fdda3f7f6baf9966f2144c5336fd79dc642cbcbe820 +size 260729 diff --git a/docs/source/_static/constraints.png b/docs/source/_static/constraints.png new file mode 100644 index 0000000000000000000000000000000000000000..e3a38c672c341b4c36bb306e41b97de021f0145c --- /dev/null +++ b/docs/source/_static/constraints.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e10d0e98940ae1dd887d345aacef62452a87d2f30ec3682e24bae78af05f0876 +size 825850 diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 0000000000000000000000000000000000000000..a6cc028387e69a0f487de9ff25cdbb3a0910a591 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,76 @@ +.hero { + padding: 2.5rem 2rem; + border-radius: 12px; + background: linear-gradient(135deg, #0f1a0c 0%, #1c2b16 55%, #76b900 100%); + color: #f8f9fb; + margin: 1.5rem 0 2rem 0; +} + +.hero-title { + font-size: 2.2rem; + margin: 0 0 0.6rem 0; +} + +.hero-subtitle { + font-size: 1.1rem; + margin: 0 0 1.2rem 0; + opacity: 0.9; +} + +.hero-actions a { + display: inline-block; + margin-right: 0.8rem; + padding: 0.5rem 0.9rem; + border-radius: 6px; + background: #76b900; + color: #0f1a0c; + text-decoration: none; + font-weight: 600; +} + +.hero-actions a.secondary { + background: transparent; + color: #f8f9fb; + border: 1px solid #f8f9fb; +} + +.card-grid { + display: grid; + gap: 1rem; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + margin: 1.5rem 0 2rem 0; +} + +.card { + border: 1px solid rgba(0, 0, 0, 0.08); + border-radius: 10px; + padding: 1rem 1.2rem; + background: #ffffff; +} + +.card h3 { + margin-top: 0; + margin-bottom: 0.4rem; +} + +.card p { + margin: 0; + color: #3c4758; +} + +.quick-links { + display: flex; + flex-wrap: wrap; + gap: 0.8rem; + margin: 1rem 0 2rem 0; +} + +.quick-links a { + display: inline-block; + padding: 0.4rem 0.8rem; + border-radius: 999px; + background: #edf2f7; + color: #1a202c; + text-decoration: none; + font-weight: 600; +} diff --git a/docs/source/_static/demo/constraints_panel.png b/docs/source/_static/demo/constraints_panel.png new file mode 100644 index 0000000000000000000000000000000000000000..7f81db13848e325d6a7c5182383aff24a5eba1b8 Binary files /dev/null and b/docs/source/_static/demo/constraints_panel.png differ diff --git a/docs/source/_static/demo/editing_mode.png b/docs/source/_static/demo/editing_mode.png new file mode 100644 index 0000000000000000000000000000000000000000..d8a966068c236cba236255827efba640578c659a --- /dev/null +++ b/docs/source/_static/demo/editing_mode.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83a049d05477ebac3be6606a081a036910baa4f3dd25369b0bf49e8d75d88c28 +size 346099 diff --git a/docs/source/_static/demo/examples_panel.png b/docs/source/_static/demo/examples_panel.png new file mode 100644 index 0000000000000000000000000000000000000000..08fa3bf4c6156d3f21fa5b028dbcccf3c601ceeb Binary files /dev/null and b/docs/source/_static/demo/examples_panel.png differ diff --git a/docs/source/_static/demo/exports_panel.png b/docs/source/_static/demo/exports_panel.png new file mode 100644 index 0000000000000000000000000000000000000000..5a6c1941048320f539cd895a62c912d177e9be1c Binary files /dev/null and b/docs/source/_static/demo/exports_panel.png differ diff --git a/docs/source/_static/demo/generate_panel.png b/docs/source/_static/demo/generate_panel.png new file mode 100644 index 0000000000000000000000000000000000000000..fe0bc75ac081ddc38f680eefd698a9b70a3afb7f Binary files /dev/null and b/docs/source/_static/demo/generate_panel.png differ diff --git a/docs/source/_static/demo/model_selection.png b/docs/source/_static/demo/model_selection.png new file mode 100644 index 0000000000000000000000000000000000000000..991d8d2b0eb460f519ceb7b252eb50250d7704d1 Binary files /dev/null and b/docs/source/_static/demo/model_selection.png differ diff --git a/docs/source/_static/demo/panel.png b/docs/source/_static/demo/panel.png new file mode 100644 index 0000000000000000000000000000000000000000..9a55cd6f56b34879b92d7eef43adc8dc67f0d266 Binary files /dev/null and b/docs/source/_static/demo/panel.png differ diff --git a/docs/source/_static/demo/timeline.png b/docs/source/_static/demo/timeline.png new file mode 100644 index 0000000000000000000000000000000000000000..aa621573ceb16eb8e140b36a720d2aa1a7cd9f9e Binary files /dev/null and b/docs/source/_static/demo/timeline.png differ diff --git a/docs/source/_static/demo/viewer.png b/docs/source/_static/demo/viewer.png new file mode 100644 index 0000000000000000000000000000000000000000..e90d3d1c8be7ece478b8e26ceac754cf5f7a79e1 --- /dev/null +++ b/docs/source/_static/demo/viewer.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efb2c64cd154547cff3c6d6fd1bcc30456836eef3f4e3f7375994e290ec16d34 +size 349903 diff --git a/docs/source/_static/hero-placeholder.svg b/docs/source/_static/hero-placeholder.svg new file mode 100644 index 0000000000000000000000000000000000000000..53458bc6032c892c0080958aad8b9cc6af3618d8 --- /dev/null +++ b/docs/source/_static/hero-placeholder.svg @@ -0,0 +1,12 @@ + + + + + + + Kimodo + + + Placeholder hero image + + diff --git a/docs/source/_static/logo-placeholder.svg b/docs/source/_static/logo-placeholder.svg new file mode 100644 index 0000000000000000000000000000000000000000..8c300944572b52223908289b0e890ac52676b344 --- /dev/null +++ b/docs/source/_static/logo-placeholder.svg @@ -0,0 +1,6 @@ + + + + Kimodo + + diff --git a/docs/source/_static/overview.png b/docs/source/_static/overview.png new file mode 100644 index 0000000000000000000000000000000000000000..71ed6db85adf7c2b27ecb0d38313b6862f157296 --- /dev/null +++ b/docs/source/_static/overview.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f30c5ff61905841d7104ebecb9ca6321426cb383eaae32a1e6676adb1fc1442d +size 469052 diff --git a/docs/source/_static/skeletons/g1.png b/docs/source/_static/skeletons/g1.png new file mode 100644 index 0000000000000000000000000000000000000000..07626569e44ce69eda1f52f7955363956eab3193 --- /dev/null +++ b/docs/source/_static/skeletons/g1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:208951c068769115c90e8675e9bfba35b14bf0d3b32be976ccae4c52ccfce213 +size 372457 diff --git a/docs/source/_static/skeletons/skeletons.png b/docs/source/_static/skeletons/skeletons.png new file mode 100644 index 0000000000000000000000000000000000000000..f38b0e99bde6905cf4b794ea44a56527852b475d --- /dev/null +++ b/docs/source/_static/skeletons/skeletons.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17ec3d418479ffc83007740caba29b92d93c615aefac9bdfe54cc1df140476ab +size 941748 diff --git a/docs/source/_static/skeletons/smplx.png b/docs/source/_static/skeletons/smplx.png new file mode 100644 index 0000000000000000000000000000000000000000..5a160e643137a0335fe0d0efda94741f036f419b --- /dev/null +++ b/docs/source/_static/skeletons/smplx.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9176048e1a8ffe64038b41e1bcd5345172f0eea140d8d9d0da19bcb43471de27 +size 370494 diff --git a/docs/source/_static/skeletons/soma.png b/docs/source/_static/skeletons/soma.png new file mode 100644 index 0000000000000000000000000000000000000000..b48e16b936700284191841cbc7f1d3a7c15bb473 --- /dev/null +++ b/docs/source/_static/skeletons/soma.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daf5e2d3c66ad129e7a63deba56620f49b232b335a72fe3bdd95a35dbdd07001 +size 398997 diff --git a/docs/source/_static/skeletons/soma_skels.png b/docs/source/_static/skeletons/soma_skels.png new file mode 100644 index 0000000000000000000000000000000000000000..659fca1d6451601bf10d0c14fe4f8ed7511cfc95 --- /dev/null +++ b/docs/source/_static/skeletons/soma_skels.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c09e3527548841f766c571b7effb57daf2729b9cd03a90a249dd3b52d02edb6 +size 827795 diff --git a/docs/source/_static/smoothed_root.png b/docs/source/_static/smoothed_root.png new file mode 100644 index 0000000000000000000000000000000000000000..8e14458016b5f7b9161c0496fe9bb839e031e225 --- /dev/null +++ b/docs/source/_static/smoothed_root.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdc013bb2d09dc68b48a9699af4d09274819225efc4c2024aea563b0fd7ca150 +size 277752 diff --git a/docs/source/_templates/apidoc/module.rst.jinja b/docs/source/_templates/apidoc/module.rst.jinja new file mode 100644 index 0000000000000000000000000000000000000000..f831f68ef1f40fbd286ed536cc742aae64396075 --- /dev/null +++ b/docs/source/_templates/apidoc/module.rst.jinja @@ -0,0 +1,16 @@ +{%- if show_headings %} +{{- [basename, "module"] | join(' ') | e | heading }} + +{% endif -%} +.. automodule:: {{ qualname }} +{%- set preferred_order = ['members', 'undoc-members', 'show-inheritance'] %} +{%- for option in preferred_order %} +{%- if option in automodule_options %} + :{{ option }}: +{%- endif %} +{%- endfor %} +{%- for option in automodule_options %} +{%- if option not in preferred_order %} + :{{ option }}: +{%- endif %} +{%- endfor %} diff --git a/docs/source/_templates/apidoc/package.rst.jinja b/docs/source/_templates/apidoc/package.rst.jinja new file mode 100644 index 0000000000000000000000000000000000000000..3a2d513f1efd6eac8ad46252e94f445a6dc05a38 --- /dev/null +++ b/docs/source/_templates/apidoc/package.rst.jinja @@ -0,0 +1,65 @@ +{%- set preferred_order = ['members', 'undoc-members', 'show-inheritance'] %} +{%- macro automodule(modname, options) -%} +.. automodule:: {{ modname }} +{%- for option in preferred_order %} +{%- if option in options %} + :{{ option }}: +{%- endif %} +{%- endfor %} +{%- for option in options %} +{%- if option not in preferred_order %} + :{{ option }}: +{%- endif %} +{%- endfor %} +{%- endmacro %} + +{%- macro toctree(docnames) -%} +.. toctree:: + :maxdepth: {{ maxdepth }} +{% for docname in docnames %} + {{ docname }} +{%- endfor %} +{%- endmacro %} + +{%- if is_namespace %} +{{- [pkgname, "namespace"] | join(" ") | e | heading }} +{% else %} +{{- [pkgname, "package"] | join(" ") | e | heading }} +{% endif %} + +{%- if is_namespace %} +.. py:module:: {{ pkgname }} +{% endif %} + +{%- if modulefirst and not is_namespace %} +{{ automodule(pkgname, automodule_options) }} +{% endif %} + +{%- if subpackages %} +Subpackages +----------- + +{{ toctree(subpackages) }} +{% endif %} + +{%- if submodules %} +Submodules +---------- +{% if separatemodules %} +{{ toctree(submodules) }} +{% else %} +{%- for submodule in submodules %} +{% if show_headings %} +{{- [submodule, "module"] | join(" ") | e | heading(2) }} +{% endif %} +{{ automodule(submodule, automodule_options) }} +{% endfor %} +{%- endif %} +{%- endif %} + +{%- if not modulefirst and not is_namespace %} +Module contents +--------------- + +{{ automodule(pkgname, automodule_options) }} +{% endif %} diff --git a/docs/source/api_reference/_generated/kimodo.demo.rst b/docs/source/api_reference/_generated/kimodo.demo.rst new file mode 100644 index 0000000000000000000000000000000000000000..b8c3b7c502e27044e45f916bc3e8cb00876bcd57 --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.demo.rst @@ -0,0 +1,69 @@ +kimodo.demo package +=================== + +Submodules +---------- + +kimodo.demo.app module +---------------------- + +.. automodule:: kimodo.demo.app + :members: + :undoc-members: + :show-inheritance: + +kimodo.demo.config module +------------------------- + +.. automodule:: kimodo.demo.config + :members: + :undoc-members: + :show-inheritance: + +kimodo.demo.embedding\_cache module +----------------------------------- + +.. automodule:: kimodo.demo.embedding_cache + :members: + :undoc-members: + :show-inheritance: + +kimodo.demo.generation module +----------------------------- + +.. automodule:: kimodo.demo.generation + :members: + :undoc-members: + :show-inheritance: + +kimodo.demo.queue\_manager module +--------------------------------- + +.. automodule:: kimodo.demo.queue_manager + :members: + :undoc-members: + :show-inheritance: + +kimodo.demo.state module +------------------------ + +.. automodule:: kimodo.demo.state + :members: + :undoc-members: + :show-inheritance: + +kimodo.demo.ui module +--------------------- + +.. automodule:: kimodo.demo.ui + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.demo + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.exports.rst b/docs/source/api_reference/_generated/kimodo.exports.rst new file mode 100644 index 0000000000000000000000000000000000000000..e7d5b8261a57ef3e4f39bc0eedb0c3b997fa93d3 --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.exports.rst @@ -0,0 +1,61 @@ +kimodo.exports package +====================== + +Submodules +---------- + +kimodo.exports.bvh module +------------------------- + +.. automodule:: kimodo.exports.bvh + :members: + :undoc-members: + :show-inheritance: + +kimodo.exports.motion\_convert\_lib module +------------------------------------------ + +.. automodule:: kimodo.exports.motion_convert_lib + :members: + :undoc-members: + :show-inheritance: + +kimodo.exports.motion\_formats module +------------------------------------- + +.. automodule:: kimodo.exports.motion_formats + :members: + :undoc-members: + :show-inheritance: + +kimodo.exports.motion\_io module +-------------------------------- + +.. automodule:: kimodo.exports.motion_io + :members: + :undoc-members: + :show-inheritance: + +kimodo.exports.mujoco module +---------------------------- + +.. automodule:: kimodo.exports.mujoco + :members: + :undoc-members: + :show-inheritance: + +kimodo.exports.smplx module +--------------------------- + +.. automodule:: kimodo.exports.smplx + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.exports + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.metrics.rst b/docs/source/api_reference/_generated/kimodo.metrics.rst new file mode 100644 index 0000000000000000000000000000000000000000..81153bf7294d6b60334b70fe3f37c94f6ee199c8 --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.metrics.rst @@ -0,0 +1,45 @@ +kimodo.metrics package +====================== + +Submodules +---------- + +kimodo.metrics.base module +-------------------------- + +.. automodule:: kimodo.metrics.base + :members: + :undoc-members: + :show-inheritance: + +kimodo.metrics.constraints module +--------------------------------- + +.. automodule:: kimodo.metrics.constraints + :members: + :undoc-members: + :show-inheritance: + +kimodo.metrics.foot\_skate module +--------------------------------- + +.. automodule:: kimodo.metrics.foot_skate + :members: + :undoc-members: + :show-inheritance: + +kimodo.metrics.tmr module +------------------------- + +.. automodule:: kimodo.metrics.tmr + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.metrics + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.model.llm2vec.models.rst b/docs/source/api_reference/_generated/kimodo.model.llm2vec.models.rst new file mode 100644 index 0000000000000000000000000000000000000000..886f24d37eea24dbfe3e564f6a096115675fc629 --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.model.llm2vec.models.rst @@ -0,0 +1,37 @@ +kimodo.model.llm2vec.models package +=================================== + +Submodules +---------- + +kimodo.model.llm2vec.models.attn\_mask\_utils module +---------------------------------------------------- + +.. automodule:: kimodo.model.llm2vec.models.attn_mask_utils + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.llm2vec.models.bidirectional\_llama module +------------------------------------------------------- + +.. automodule:: kimodo.model.llm2vec.models.bidirectional_llama + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.llm2vec.models.utils module +---------------------------------------- + +.. automodule:: kimodo.model.llm2vec.models.utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.model.llm2vec.models + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.model.llm2vec.rst b/docs/source/api_reference/_generated/kimodo.model.llm2vec.rst new file mode 100644 index 0000000000000000000000000000000000000000..94be3aa429b0bb1d317eb0f6988378cee5f84965 --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.model.llm2vec.rst @@ -0,0 +1,37 @@ +kimodo.model.llm2vec package +============================ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + kimodo.model.llm2vec.models + +Submodules +---------- + +kimodo.model.llm2vec.llm2vec module +----------------------------------- + +.. automodule:: kimodo.model.llm2vec.llm2vec + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.llm2vec.llm2vec\_wrapper module +-------------------------------------------- + +.. automodule:: kimodo.model.llm2vec.llm2vec_wrapper + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.model.llm2vec + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.model.rst b/docs/source/api_reference/_generated/kimodo.model.rst new file mode 100644 index 0000000000000000000000000000000000000000..a5e57dffce12586789dc890e9bbc5eee13082a11 --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.model.rst @@ -0,0 +1,109 @@ +kimodo.model package +==================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + kimodo.model.llm2vec + +Submodules +---------- + +kimodo.model.backbone module +---------------------------- + +.. automodule:: kimodo.model.backbone + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.cfg module +----------------------- + +.. automodule:: kimodo.model.cfg + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.common module +-------------------------- + +.. automodule:: kimodo.model.common + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.diffusion module +----------------------------- + +.. automodule:: kimodo.model.diffusion + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.kimodo\_model module +--------------------------------- + +.. automodule:: kimodo.model.kimodo_model + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.load\_model module +------------------------------- + +.. automodule:: kimodo.model.load_model + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.loading module +--------------------------- + +.. automodule:: kimodo.model.loading + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.registry module +---------------------------- + +.. automodule:: kimodo.model.registry + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.text\_encoder\_api module +-------------------------------------- + +.. automodule:: kimodo.model.text_encoder_api + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.tmr module +----------------------- + +.. automodule:: kimodo.model.tmr + :members: + :undoc-members: + :show-inheritance: + +kimodo.model.twostage\_denoiser module +-------------------------------------- + +.. automodule:: kimodo.model.twostage_denoiser + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.model + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.motion_rep.reps.rst b/docs/source/api_reference/_generated/kimodo.motion_rep.reps.rst new file mode 100644 index 0000000000000000000000000000000000000000..3d6746ef6acd78b2ec48ebe83c1e3d737237f93c --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.motion_rep.reps.rst @@ -0,0 +1,37 @@ +kimodo.motion\_rep.reps package +=============================== + +Submodules +---------- + +kimodo.motion\_rep.reps.base module +----------------------------------- + +.. automodule:: kimodo.motion_rep.reps.base + :members: + :undoc-members: + :show-inheritance: + +kimodo.motion\_rep.reps.kimodo\_motionrep module +------------------------------------------------ + +.. automodule:: kimodo.motion_rep.reps.kimodo_motionrep + :members: + :undoc-members: + :show-inheritance: + +kimodo.motion\_rep.reps.tmr\_motionrep module +--------------------------------------------- + +.. automodule:: kimodo.motion_rep.reps.tmr_motionrep + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.motion_rep.reps + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.motion_rep.rst b/docs/source/api_reference/_generated/kimodo.motion_rep.rst new file mode 100644 index 0000000000000000000000000000000000000000..a7c1b1a176c6e9c1e4909dbf826d5f84b589841b --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.motion_rep.rst @@ -0,0 +1,61 @@ +kimodo.motion\_rep package +========================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + kimodo.motion_rep.reps + +Submodules +---------- + +kimodo.motion\_rep.conditioning module +-------------------------------------- + +.. automodule:: kimodo.motion_rep.conditioning + :members: + :undoc-members: + :show-inheritance: + +kimodo.motion\_rep.feature\_utils module +---------------------------------------- + +.. automodule:: kimodo.motion_rep.feature_utils + :members: + :undoc-members: + :show-inheritance: + +kimodo.motion\_rep.feet module +------------------------------ + +.. automodule:: kimodo.motion_rep.feet + :members: + :undoc-members: + :show-inheritance: + +kimodo.motion\_rep.smooth\_root module +-------------------------------------- + +.. automodule:: kimodo.motion_rep.smooth_root + :members: + :undoc-members: + :show-inheritance: + +kimodo.motion\_rep.stats module +------------------------------- + +.. automodule:: kimodo.motion_rep.stats + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.motion_rep + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.rst b/docs/source/api_reference/_generated/kimodo.rst new file mode 100644 index 0000000000000000000000000000000000000000..32392ddad287bd76804b9c102bdf9003981baf3c --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.rst @@ -0,0 +1,84 @@ +kimodo package +============== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + kimodo.demo + kimodo.exports + kimodo.metrics + kimodo.model + kimodo.motion_rep + kimodo.scripts + kimodo.skeleton + kimodo.viz + +Submodules +---------- + +kimodo.assets module +-------------------- + +.. automodule:: kimodo.assets + :members: + :undoc-members: + :show-inheritance: + +kimodo.constraints module +------------------------- + +.. automodule:: kimodo.constraints + :members: + :undoc-members: + :show-inheritance: + +kimodo.geometry module +---------------------- + +.. automodule:: kimodo.geometry + :members: + :undoc-members: + :show-inheritance: + +kimodo.meta module +------------------ + +.. automodule:: kimodo.meta + :members: + :undoc-members: + :show-inheritance: + +kimodo.postprocess module +------------------------- + +.. automodule:: kimodo.postprocess + :members: + :undoc-members: + :show-inheritance: + +kimodo.sanitize module +---------------------- + +.. automodule:: kimodo.sanitize + :members: + :undoc-members: + :show-inheritance: + +kimodo.tools module +------------------- + +.. automodule:: kimodo.tools + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.scripts.rst b/docs/source/api_reference/_generated/kimodo.scripts.rst new file mode 100644 index 0000000000000000000000000000000000000000..a88121896f315575755181733b9be4136ce0b478 --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.scripts.rst @@ -0,0 +1,61 @@ +kimodo.scripts package +====================== + +Submodules +---------- + +kimodo.scripts.generate module +------------------------------ + +.. automodule:: kimodo.scripts.generate + :members: + :undoc-members: + :show-inheritance: + +kimodo.scripts.gradio\_theme module +----------------------------------- + +.. automodule:: kimodo.scripts.gradio_theme + :members: + :undoc-members: + :show-inheritance: + +kimodo.scripts.lock\_requirements module +---------------------------------------- + +.. automodule:: kimodo.scripts.lock_requirements + :members: + :undoc-members: + :show-inheritance: + +kimodo.scripts.motion\_convert module +------------------------------------- + +.. automodule:: kimodo.scripts.motion_convert + :members: + :undoc-members: + :show-inheritance: + +kimodo.scripts.mujoco\_load module +---------------------------------- + +.. automodule:: kimodo.scripts.mujoco_load + :members: + :undoc-members: + :show-inheritance: + +kimodo.scripts.run\_text\_encoder\_server module +------------------------------------------------ + +.. automodule:: kimodo.scripts.run_text_encoder_server + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.scripts + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.skeleton.rst b/docs/source/api_reference/_generated/kimodo.skeleton.rst new file mode 100644 index 0000000000000000000000000000000000000000..0398979c108b5a1bc874da579c2a3854b4f2d88d --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.skeleton.rst @@ -0,0 +1,61 @@ +kimodo.skeleton package +======================= + +Submodules +---------- + +kimodo.skeleton.base module +--------------------------- + +.. automodule:: kimodo.skeleton.base + :members: + :undoc-members: + :show-inheritance: + +kimodo.skeleton.bvh module +-------------------------- + +.. automodule:: kimodo.skeleton.bvh + :members: + :undoc-members: + :show-inheritance: + +kimodo.skeleton.definitions module +---------------------------------- + +.. automodule:: kimodo.skeleton.definitions + :members: + :undoc-members: + :show-inheritance: + +kimodo.skeleton.kinematics module +--------------------------------- + +.. automodule:: kimodo.skeleton.kinematics + :members: + :undoc-members: + :show-inheritance: + +kimodo.skeleton.registry module +------------------------------- + +.. automodule:: kimodo.skeleton.registry + :members: + :undoc-members: + :show-inheritance: + +kimodo.skeleton.transforms module +--------------------------------- + +.. automodule:: kimodo.skeleton.transforms + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.skeleton + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/kimodo.viz.rst b/docs/source/api_reference/_generated/kimodo.viz.rst new file mode 100644 index 0000000000000000000000000000000000000000..f840339369957a3b4ff7303407c6b5ef14db3c20 --- /dev/null +++ b/docs/source/api_reference/_generated/kimodo.viz.rst @@ -0,0 +1,93 @@ +kimodo.viz package +================== + +Submodules +---------- + +kimodo.viz.constraint\_ui module +-------------------------------- + +.. automodule:: kimodo.viz.constraint_ui + :members: + :undoc-members: + :show-inheritance: + +kimodo.viz.coords module +------------------------ + +.. automodule:: kimodo.viz.coords + :members: + :undoc-members: + :show-inheritance: + +kimodo.viz.g1\_rig module +------------------------- + +.. automodule:: kimodo.viz.g1_rig + :members: + :undoc-members: + :show-inheritance: + +kimodo.viz.gui module +--------------------- + +.. automodule:: kimodo.viz.gui + :members: + :undoc-members: + :show-inheritance: + +kimodo.viz.playback module +-------------------------- + +.. automodule:: kimodo.viz.playback + :members: + :undoc-members: + :show-inheritance: + +kimodo.viz.scene module +----------------------- + +.. automodule:: kimodo.viz.scene + :members: + :undoc-members: + :show-inheritance: + +kimodo.viz.smplx\_skin module +----------------------------- + +.. automodule:: kimodo.viz.smplx_skin + :members: + :undoc-members: + :show-inheritance: + +kimodo.viz.soma\_layer\_skin module +----------------------------------- + +.. automodule:: kimodo.viz.soma_layer_skin + :members: + :undoc-members: + :show-inheritance: + +kimodo.viz.soma\_skin module +---------------------------- + +.. automodule:: kimodo.viz.soma_skin + :members: + :undoc-members: + :show-inheritance: + +kimodo.viz.viser\_utils module +------------------------------ + +.. automodule:: kimodo.viz.viser_utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: kimodo.viz + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/_generated/modules.rst b/docs/source/api_reference/_generated/modules.rst new file mode 100644 index 0000000000000000000000000000000000000000..51a526e47cbf6cabb352030d51b2ad29671623b6 --- /dev/null +++ b/docs/source/api_reference/_generated/modules.rst @@ -0,0 +1,7 @@ +kimodo +====== + +.. toctree:: + :maxdepth: 4 + + kimodo diff --git a/docs/source/api_reference/constraints.rst b/docs/source/api_reference/constraints.rst new file mode 100644 index 0000000000000000000000000000000000000000..3251d3c31d78e38401aa0c70e61d73052d8f59db --- /dev/null +++ b/docs/source/api_reference/constraints.rst @@ -0,0 +1,9 @@ +Constraints +=========== + +Constraint definitions and utilities. + +.. automodule:: kimodo.constraints + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/exports.rst b/docs/source/api_reference/exports.rst new file mode 100644 index 0000000000000000000000000000000000000000..9402063923de17529bb8bd7c5d1e83d0c4a01913 --- /dev/null +++ b/docs/source/api_reference/exports.rst @@ -0,0 +1,19 @@ +Exports +======= + +Export utilities for common formats. + +.. automodule:: kimodo.exports.bvh + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: kimodo.exports.mujoco + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: kimodo.exports.smplx + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/index.rst b/docs/source/api_reference/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..eed22e00f38e65d4a5313a24257e84fd23610011 --- /dev/null +++ b/docs/source/api_reference/index.rst @@ -0,0 +1,16 @@ +API Reference +============= + +This section contains the API documentation for Kimodo, organized by domain. + +.. toctree:: + :maxdepth: 2 + :caption: Core Modules + + model + motion_rep + constraints + exports + viz + utilities + post-processing diff --git a/docs/source/api_reference/model.rst b/docs/source/api_reference/model.rst new file mode 100644 index 0000000000000000000000000000000000000000..eacc8d616cac9f0fb80f3d0317664e56c6bf811d --- /dev/null +++ b/docs/source/api_reference/model.rst @@ -0,0 +1,45 @@ +Model +===== + +Core model architecture, diffusion logic, and text encoders. + + +Kimodo Model +------------ + +.. automodule:: kimodo.model.kimodo_model + :members: + :undoc-members: + :special-members: __call__ + +Denoiser and Backbone +--------------------- +.. automodule:: kimodo.model.twostage_denoiser + :members: + :undoc-members: + +.. automodule:: kimodo.model.backbone + :members: + :undoc-members: + +Classifier-Free Guidance +------------------------ + +.. automodule:: kimodo.model.cfg + :members: + :undoc-members: + +Model Loading +------------- + +.. automodule:: kimodo.model.load_model + :members: + :undoc-members: + +Text Encoder +------------ + +.. automodule:: kimodo.model.text_encoder_api + :members: + :undoc-members: + :special-members: __call__ diff --git a/docs/source/api_reference/motion_rep.rst b/docs/source/api_reference/motion_rep.rst new file mode 100644 index 0000000000000000000000000000000000000000..3d06fac6641eb856ab3166f0f97850fc3159bb6e --- /dev/null +++ b/docs/source/api_reference/motion_rep.rst @@ -0,0 +1,56 @@ +Motion Representation +===================== + +Motion representation utilities and kinematics helpers. + +Skeleton +-------- + +.. automodule:: kimodo.skeleton + :members: + :undoc-members: + :show-inheritance: + +Forward Kinematics +------------------ + +.. automodule:: kimodo.skeleton.kinematics + :members: + :undoc-members: + :show-inheritance: + +Motion Representations +---------------------- + +.. automodule:: kimodo.motion_rep.reps.base + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: kimodo.motion_rep.reps.kimodo_motionrep + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: kimodo.motion_rep.reps.tmr_motionrep + :members: + :undoc-members: + :show-inheritance: + +Utilities +--------- + +.. automodule:: kimodo.motion_rep.feet + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: kimodo.motion_rep.stats + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: kimodo.motion_rep.smooth_root + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/post-processing.rst b/docs/source/api_reference/post-processing.rst new file mode 100644 index 0000000000000000000000000000000000000000..00485886adae0aca6dacc815be8f604969c3fe3d --- /dev/null +++ b/docs/source/api_reference/post-processing.rst @@ -0,0 +1,7 @@ +Post-Processing Bindings +======================== + +.. automodule:: kimodo.postprocess + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/utilities.rst b/docs/source/api_reference/utilities.rst new file mode 100644 index 0000000000000000000000000000000000000000..96f3f6bcb977b41d97343d63cb23cba37378a266 --- /dev/null +++ b/docs/source/api_reference/utilities.rst @@ -0,0 +1,19 @@ +Utilities +========= + +General utilities used across the codebase. + +.. automodule:: kimodo.tools + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: kimodo.geometry + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: kimodo.sanitize + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_reference/viz.rst b/docs/source/api_reference/viz.rst new file mode 100644 index 0000000000000000000000000000000000000000..9a74f37705b1782f6c81d3fbadcea9f5719ffe5e --- /dev/null +++ b/docs/source/api_reference/viz.rst @@ -0,0 +1,19 @@ +Visualization +============= + +Visualization helpers for rendering skeletons and meshes. + +.. automodule:: kimodo.viz.g1_rig + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: kimodo.viz.smplx_skin + :members: + :undoc-members: + :show-inheritance: + +.. automodule:: kimodo.viz.viser_utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/benchmark/introduction.md b/docs/source/benchmark/introduction.md new file mode 100644 index 0000000000000000000000000000000000000000..1799bf91c0fae135dfa610a79fcc27b384a549cd --- /dev/null +++ b/docs/source/benchmark/introduction.md @@ -0,0 +1,141 @@ +# Benchmark Introduction + +We provide a benchmark to evaluate text-to-motion and constrained motion generation on a shared test suite. +For reproducibility, all test content is stored on disk as folders and files, so anyone can run exactly the same cases. +The benchmark test suite is available to download from HuggingFace at [`nvidia/Kimodo-Motion-Gen-Benchmark`](https://huggingface.co/datasets/nvidia/Kimodo-Motion-Gen-Benchmark) and is currently set up for use with models trained on the [SOMA](https://github.com/NVlabs/SOMA-X) body skeleton. + +The benchmark contains text prompts, durations, and constraint configurations for a variety of test cases, but **not** the ground-truth motion data itself. The ground-truth motions are derived from the [BONES-SEED dataset](https://huggingface.co/datasets/bones-studio/seed), which has its own license you should consider. So to construct the full benchmark motions, you must download the BONES-SEED dataset separately and run our `create_benchmark` script to populate the test suite with ground-truth motions. + +Constructing the benchmark with `create_benchmark` is the first step in the full [Evaluation Pipeline](pipeline.md), which is described in detail on the next page. In addition to the benchmark test cases, we provide code to run generation with Kimodo and compute a variety of [metrics](metrics.md) measuring motion quality, text alignment, and constraint following. While this open-sourced public test suite is not the exact same used in the [Kimodo tech report](https://research.nvidia.com/labs/sil/projects/kimodo/assets/kimodo_tech_report.pdf) (Sec. 6.1), the evaluation metrics are the same and evaluation methodology is similar. + +On this page, we describe the overall structure of the test suite and details of the different test cases. Then in subsequent pages, we describe how to run the full [evaluation pipeline](pipeline.md), detail the [metrics](metrics.md), and finally provide the [results](results.md) of Kimodo-SOMA-RP and Kimodo-SOMA-SEED on the benchmark. + +## Dataset Splits +To evaluate a model on the benchmark, it should be trained with the [provided splits](https://huggingface.co/datasets/nvidia/Kimodo-Motion-Gen-Benchmark/tree/main/splits) for the [BONES-SEED dataset](https://huggingface.co/datasets/bones-studio/seed). + +The different splits are defined in: + +- `train_split_paths.txt` - filenames of training data +- `test_content_split_paths.txt` - filenames for test split containing new semantic "content". This split contains motions with `content_name` (from the BONES-SEED metadata) that are not seen in the training split. This tests model generalization to new semantic motion types, e.g. for text-to-motion generalization. +- `test_repetition_split_paths.txt` - filenames for test split containing new motions from content that was seen in training. This split contains motions where the `content_name` is contained in the training split, but the exact motion itself was not seen. This tests a model's ability to generalize to novel performances of a familiar motion type, e.g., for constraint-following generalization. + +The training split should be used for training, while the two test splits (`content` and `repetition`) are used in the test suite, as described below. Note that the test cases in the benchmark do not cover the entire content and repetition test splits, instead we strategically sample a subset that maximizes content diversity. + +## Test Suite Structure + +The full test suite contains 22,474 test cases spanning text and constraint-conditioned motion generation. +The suite is organized hierarchically to logically group together test cases, so the evaluation pipeline can be run on a subset of the benchmark instead of the full thing, if desired. + +After the benchmark has been constructed and motions generated for the model to evaluate, a **test case** is a single folder containing: + +- `meta.json` (**required**): text prompt(s) and duration(s), +- `constraints.json` (**optional**): constraints for controlled generation, using the [constraints format](../user_guide/constraints.md), +- `gt_motion.npz` (**optional**): ground-truth/reference motion, using the [NPZ output format](../user_guide/output_formats.md), +- `motion.npz` (**optional**): output of the model given the `meta.json` prompt/duration and optional `constraints.json`, using the same [NPZ output format](../user_guide/output_formats.md). + +In addition to being used in the evaluation pipeline, each test case can be: + +- loaded in the interactive demo through **Load Example** for visualization, +- loaded in `kimodo_gen` with `--input_folder` for generation from folder-defined inputs. + +### Benchmark Folder Hierarchy + +The full suite is organized as follows: + +```text +testsuite +├── content +│ ├── constraints_notext +│ │ ├── end-effectors +│ │ ├── fullbody +│ │ ├── mixture +│ │ └── root +│ ├── constraints_withtext +│ │ ├── end-effectors +│ │ ├── fullbody +│ │ ├── mixture +│ │ └── root +│ └── text2motion +│ ├── overview +│ ├── timeline_multi +│ └── timeline_single +└── repetition + ├── constraints_notext + │ ├── end-effectors + │ ├── fullbody + │ ├── mixture + │ └── root + ├── constraints_withtext + │ ├── end-effectors + │ ├── fullbody + │ ├── mixture + │ └── root + └── text2motion + ├── overview + ├── timeline_multi + └── timeline_single +``` + +At the highest level, the test suite is organized by the test split used. As discussed previously, `content` refers to the test split with held out semantic categories of motion, while `repetition` refers to held out motions from semantic categories seen during training. + +Within each test split, test cases are organized into: + +* `text2motion`: test cases with only text prompts as input (no constraints) +* `constraints_notext`: test cases with only constraints as input (no text prompt) +* `constraints_withtext`: test cases with both prompt and constraints as input + +### Text2Motion Test Cases + +These test cases are pure text-to-motion with no constraints as input. `text2motion` test cases exclusively use prompts derived from our [SEED timeline annotations](https://huggingface.co/datasets/nvidia/SEED-Timeline-Annotations). It contains three types of test cases: + +* `overview`: medium-detail prompt that describes a full motion. Corresponds to `overview_description` in the [NVIDIA SEED timelines](https://huggingface.co/datasets/nvidia/SEED-Timeline-Annotations) or equivalently `content_natural_desc_4` in the [BONES SEED](https://huggingface.co/datasets/bones-studio/seed) metadata. +* `timeline_single`: fine-grained prompt describing a single segment of a timeline annotation. Corresponds to a single event in a SEED timeline. +* `timeline_multi`: fine-grained prompt describing multiple subsequent segments of a timeline annotation. Corresponds to multiple contiguous events in a SEED timeline, which have been concatenated with an LLM to get a single natural text description. + +### Constrained Test Cases + +Constrained test cases provide a constraint input either without a text prompt (i.e., `constraints_notext`) or with an `overview` text prompt (i.e., `constraints_withtext`). The different types of constraint categories mirror the [constraint types support by Kimodo](../key_concepts/constraints.md) and include: + +* `fullbody`: constrains all joint positions in the skeleton at specific frames +* `end-effectors`: constraints the position and rotations of hand and/or feet joints at specific frames +* `root`: constraints the 2D root position and optionally heading on a path or at specific frames +* `mixture`: evaluates compositional control when multiple constraint families are combined + +Within each constraint type in the hierarchy are multiple subtypes that vary the constraint sparsity patterns (either in time or in space). So the hierarchy of a `constraint` folder is: + +```text +constraints_XX +├── end-effectors +│ ├── feet_posrot # feet only constraints +│ ├── hands_feet_posrot # hands + feet constraints +│ └── hands_posrot # hands only constraints +├── fullbody +│ ├── inbetweening # constraints at start and end only +│ └── random # constraints at random frames +├── mixture +│ ├── root_ee_hands_feet_posrot_fullbody # mix of (1) root trajectory, (2) hand + foot, and (3) full-body +│ ├── root_ee_hands_posrot # mix of (1) root keyframe, and (2) hands +│ ├── root_ee_hands_posrot_fullbody # mix of (1) root keyframe, (2) hands, and (3) full-body +│ └── root_path_fullbody # mix of (1) root trajectory, and (2) full-body +└── root + ├── path_2dpos # root trajectory position + ├── path_2dposrot # root trajecotry position + heading + ├── waypoint_2dpos # root waypoint position + └── waypoint_2dposrot # root waypoint position + heading +``` + +### Indexed Test Cases in Leaf Folders + +Each leaf folder contains indexed test cases (`0000`, `0001`, `0002`, ...). +For example: + +```text +end-effectors/feet_posrot/ +├── 0000/ +├── 0001/ +├── 0002/ +... +└── 0255/ +``` + +Each index folder is one standalone test case with its own `meta.json`, optional `constraints.json`, optional `gt_motion.npz`, and optional `motion.npz`. diff --git a/docs/source/benchmark/metrics.md b/docs/source/benchmark/metrics.md new file mode 100644 index 0000000000000000000000000000000000000000..0193b2f22eb9ee2ee9742b85dcd30a3569dbb669 --- /dev/null +++ b/docs/source/benchmark/metrics.md @@ -0,0 +1,259 @@ +# Metrics + +The benchmark evaluates generated motion along three axes: + +- **Motion quality** -- foot-skate and contact-consistency metrics, +- **Constraint following** -- position error for root, end-effector, and full-body constraints, +- **Text alignment** -- TMR retrieval and distributional metrics. + +Metrics are implemented in `kimodo/metrics/` and orchestrated by `benchmark/evaluate_folder.py`. +The protocol is aligned with the [tech report](https://research.nvidia.com/labs/sil/projects/kimodo/assets/kimodo_tech_report.pdf) (Sec. 6.1, "Evaluation Metrics"). + +## Evaluation Protocol + +The evaluation pipeline runs two passes over each group of test cases: + +1. **Generated pass** -- evaluates `motion.npz` with all metrics (foot skate, contact consistency, constraint following) and, when TMR embeddings are available, computes retrieval and FID scores. +2. **Ground-truth pass** -- evaluates `gt_motion.npz` with the same motion-quality and constraint metrics. TMR retrieval metrics are not recomputed in this pass. + +Running both passes enables side-by-side comparison: the GT row serves as an empirical upper bound for motion quality, and deviations between GT and generated metrics highlight where the model can improve. See [Evaluation pipeline](pipeline.md) for the full workflow. + +## Metrics Reference + +The table below lists every key written to `metrics.json`. Detailed descriptions follow in subsequent sections. + +| Key | Category | Unit | Direction | +| --- | --- | --- | --- | +| `foot_skate_from_height` | Motion quality | m/s | Lower is better | +| `foot_skate_from_pred_contacts` | Motion quality | m/s | Lower is better | +| `foot_skate_max_vel` | Motion quality | m/s | Lower is better | +| `foot_skate_ratio` | Motion quality | ratio (0--1) | Lower is better | +| `foot_contact_consistency` | Motion quality | ratio (0--1) | Higher is better | +| `constraint_root2d_err` | Constraint follow | m | Lower is better | +| `constraint_root2d_err_p95` | Constraint follow | m | Lower is better | +| `constraint_root2d_acc` | Constraint follow | ratio (0--1) | Higher is better | +| `constraint_fullbody_keyframe` | Constraint follow | m | Lower is better | +| `constraint_end_effector` | Constraint follow | m | Lower is better | +| `TMR/t2m_sim` | Text alignment | score (0--1) | Higher is better | +| `TMR/t2m_R/R01` ... `R10` | Text alignment | % | Higher is better | +| `TMR/t2m_R/MedR` | Text alignment | rank | Lower is better | +| `TMR/FID/gen_text` | Text alignment | distance | Lower is better | +| `TMR/FID/gen_gt` | Text alignment | distance | Lower is better | +| `TMR/FID/gt_text` | Text alignment | distance | Lower is better | +| `TMR/m2m_sim` | Text alignment | score (0--1) | Higher is better | +| `TMR/t2m_gt_sim` | Text alignment | score (0--1) | Higher is better | +| `TMR/m2m_R/R01` ... `R10` | Text alignment | % | Higher is better | +| `TMR/t2m_gt_R/R01` ... `R10` | Text alignment | % | Higher is better | + +:::{note} +Raw metric values are stored in SI units (meters for positions, m/s for velocities). +The summary tables printed by `benchmark/parse_folder.py` convert constraint position errors to **cm** and foot-skate velocities to **cm/s** for readability. +::: + +### Foot Skating Metrics + +Foot skating measures how much a foot slides along the ground when it should be in static contact with the ground. Four complementary metrics capture different aspects of this artifact. + +- **`foot_skate_from_height`** (m/s, lower is better): + Mean velocity of the **toe joints** (left toe, right toe) on frames where the toe height is below a floor threshold (`height_thresh = 0.05 m`). + This metric does not rely on predicted contact labels -- it uses a geometric criterion (Y-coordinate < threshold) to identify ground-contact frames. + +- **`foot_skate_from_pred_contacts`** (m/s, lower is better): + Mean velocity of all **four foot joints** (left/right heel and toe) on frames where the model predicts contact via the `foot_contacts` output. + Unlike `foot_skate_from_height`, this metric trusts the model's own contact predictions and measures all four foot joints rather than toes only. + +- **`foot_skate_max_vel`** (m/s, lower is better): + Maximum velocity across all four foot joints and all time steps where predicted contact is active. + This captures worst-case slip spikes that mean-based metrics can hide. + +- **`foot_skate_ratio`** (ratio 0--1, lower is better): + Fraction of ground-contact frames where toe velocity exceeds a threshold (`vel_thresh = 0.2 m/s`). A frame counts as ground contact when the toe is below `height_thresh = 0.05 m` on both the current and the next frame. Inspired by the [GMD](https://github.com/korrawe/guided-motion-diffusion) skating metric. + +### Contact Consistency Metric + +- **`foot_contact_consistency`** (ratio 0--1, higher is better): + Agreement between the model's predicted foot contacts and a heuristic contact detector based on joint height and velocity (`vel_thresh = 0.15 m/s`, `height_thresh = 0.10 m`). + Computed as accuracy (`1 - incorrect_ratio`) over all time steps and four contact channels. + A score of 1.0 means perfect agreement between predicted and heuristic contacts. + As noted in the [tech report](https://research.nvidia.com/labs/sil/projects/kimodo/assets/kimodo_tech_report.pdf), this metric provides important context for interpreting the contact-based foot-skate metrics above: if contact consistency is low, `foot_skate_from_pred_contacts` may be unreliable. + +### Constraint-Following Metrics + +Constraint metrics are computed only when the test case includes a `constraints.json` file. The `ContraintFollow` metric class dispatches by [constraint type](../key_concepts/constraints.md): + +- **`constraint_end_effector`** (m, lower is better): + Mean Euclidean distance between target end-effector positions and generated joint positions at the constrained frames. + Only position-constrained joints are evaluated (rotation targets are not measured by this metric). + +- **`constraint_fullbody_keyframe`** (m, lower is better): + Mean per-joint Euclidean distance between target and generated full-body joint positions at keyframes. + The error is averaged over all joints and all keyframe frames. + +- **`constraint_root2d_err`** (m, lower is better): + Mean 2D Euclidean distance (in the XZ ground plane) between target and generated root positions at constrained frames. + +- **`constraint_root2d_err_p95`** (m, lower is better): + 95th percentile of the per-frame root 2D error across all samples in a group. + Computed during aggregation by `evaluate_folder.py` to capture tail-end failures that the mean can mask. + +- **`constraint_root2d_acc`** (ratio 0--1, higher is better): + Fraction of constrained root frames where the 2D position error is within a distance threshold (`root_threshold = 0.10 m`). + +### TMR-Based Metrics + +Text alignment is evaluated using [TMR](https://mathis.petrovich.fr/tmr/) (Text-to-Motion Retrieval), a separate encoder model that maps both text and motion into a shared embedding space. TMR is not used for generation -- it is loaded only for evaluation (see `kimodo/model/tmr.py`). + +We release a version of TMR retrained on the full Rigplay dataset as [`TMR-SOMA-RP-v1`](https://huggingface.co/nvidia/TMR-SOMA-RP-v1). The original TMR was trained on HumanML3D; our retrained variant uses the same architecture but is trained on the Rigplay motion dataset, SOMA skeleton, and with [LLM2Vec](https://github.com/McGill-NLP/llm2vec) text embeddings. + +#### Similarity Scores + +TMR encodes each text prompt and each motion clip into a unit-length embedding vector. Cosine similarity between text and motion embeddings is rescaled to a [0, 1] range: + +``` +score = cosine_similarity / 2 + 0.5 +``` + +Three per-test-case similarity scores are recorded: + +- **`TMR/t2m_sim`** (0--1, higher is better): similarity between the text prompt and the generated motion. +- **`TMR/m2m_sim`** (0--1, higher is better): similarity between the generated and ground-truth motions (only when GT is available). +- **`TMR/t2m_gt_sim`** (0--1, higher is better): similarity between the text prompt and the GT motion (only when GT is available). + +#### R-precision (Retrieval Accuracy) + +R-precision measures whether the correct motion can be retrieved from a pool given its corresponding text query. +For each text query in the evaluation group, all motions are ranked by TMR similarity. +R@k is the percentage of queries where the correct motion appears in the top k results. + +Reported keys: `TMR/t2m_R/R01`, `R02`, `R03`, `R05`, `R10` (%), and `TMR/t2m_R/MedR` (median rank, lower is better) correspond to retrieval accuracy when using generated motions. + +When ground-truth motions are available, analogous retrieval metrics are computed for motion-to-GT-motion (`TMR/m2m_R/...`) and text-to-GT-motion (`TMR/t2m_gt_R/...`). + +:::{note} +Near-duplicate text prompts can artificially penalize retrieval ranking. The evaluation handles this by grouping prompts whose text-text similarity exceeds a threshold of 0.99 and treating any motion in that group as a valid match. +::: + +#### FID (Frechet Inception Distance) + +FID measures distributional distance between two sets of TMR embeddings by fitting a multivariate Gaussian to each set and computing the Frechet distance. Three FID variants are reported: + +- **`TMR/FID/gen_gt`**: distance between generated-motion and GT-motion embeddings (only when GT is available). This is the FID metric that is typically reported in the motion generation literature. +- **`TMR/FID/gen_text`**: distance between generated-motion embeddings and text embeddings. +- **`TMR/FID/gt_text`**: distance between GT-motion and text embeddings (only when GT is available). + +Lower values indicate that the two distributions are more similar. FID requires at least 2 samples; groups with fewer samples report `NaN`. + +#### Per-Test-Case Retrieval + +In addition to the aggregate metrics above, each test case's `metrics.json` includes a `tmr` block with single motion retrieval results: + +- `t2m_rank`: the rank of the correct motion when retrieving with this test case's text query. +- `top5_retrieved`: the top-5 retrieved motions (sample IDs and text prompts) for inspection. + +## JSON Output Format + +Below is a representative `metrics.json` written by `evaluate_folder.py` for a single test case with mixed constraints (root + end-effector + full-body) and TMR embeddings: + +```json +{ + "num_motions": 1, + "folder": "...", + "per_motion_mean_gen": { + "foot_skate_from_height": 0.3144, + "foot_skate_from_pred_contacts": 0.0672, + "foot_skate_max_vel": 0.2109, + "foot_contact_consistency": 0.9522, + "foot_skate_ratio": 0.2182, + "constraint_end_effector": 0.0286, + "constraint_root2d_err": 0.0534, + "constraint_root2d_acc": 1.0, + "constraint_fullbody_keyframe": 0.0324, + "TMR/t2m_sim": 0.8209 + }, + "per_motion_mean_gt": { + "foot_skate_from_height": 0.2361, + "foot_skate_from_pred_contacts": 0.0269, + "foot_skate_max_vel": 0.1459, + "foot_contact_consistency": 1.0, + "foot_skate_ratio": 0.1402, + "constraint_end_effector": 9.82e-07, + "constraint_root2d_err": 0.0407, + "constraint_root2d_acc": 1.0, + "constraint_fullbody_keyframe": 8.73e-07 + }, + "tmr": { + "t2m_rank": 2, + "text": "A person is swiftly performing a dance move by moving their hands and legs.", + "top5_retrieved": [ + { + "id": "0231", + "text": "A person is performing dance steps while stepping back and forward..." + }, + { + "id": "0029", + "text": "A person is swiftly performing a dance move by moving their hands and legs." + } + ] + } +} +``` + +Group-level aggregate JSONs (`.json`) have the same structure but with `num_motions > 1`, averaged per-motion metrics, additional keys like `constraint_root2d_err_p95`, and a `tmr` block containing the aggregate retrieval and FID scores: + +```json +{ + "num_motions": 256, + "folder": "...", + "per_motion_mean_gen": { + "foot_skate_from_height": 0.1742, + "foot_skate_from_pred_contacts": 0.0611, + "foot_skate_max_vel": 0.3747, + "foot_contact_consistency": 0.9483, + "foot_skate_ratio": 0.1499, + "constraint_end_effector": 0.0367, + "constraint_root2d_err": 0.0495, + "constraint_root2d_acc": 0.9212, + "constraint_fullbody_keyframe": 0.0324, + "constraint_root2d_err_p95": 0.1115 + }, + "per_motion_mean_gt": { + "foot_skate_from_height": 0.1617, + "foot_skate_from_pred_contacts": 0.0235, + "foot_skate_max_vel": 0.1185, + "foot_contact_consistency": 1.0, + "foot_skate_ratio": 0.1214, + "constraint_end_effector": 1.48e-06, + "constraint_root2d_err": 0.0376, + "constraint_root2d_acc": 1.0, + "constraint_fullbody_keyframe": 1.16e-06, + "constraint_root2d_err_p95": 0.0602 + }, + "tmr": { + "TMR/t2m_sim": 0.8742, + "TMR/t2m_R/R01": 75.39, + "TMR/t2m_R/R02": 85.55, + "TMR/t2m_R/R03": 88.28, + "TMR/t2m_R/R05": 90.23, + "TMR/t2m_R/R10": 93.36, + "TMR/t2m_R/MedR": 1.0, + "TMR/t2m_R/len": 256.0, + "TMR/FID/gen_text": 0.1442, + "TMR/m2m_R/R01": 94.53, + "TMR/m2m_R/R02": 97.66, + "TMR/m2m_R/R03": 98.05, + "TMR/m2m_R/R05": 98.83, + "TMR/m2m_R/R10": 99.22, + "TMR/m2m_R/MedR": 1.0, + "TMR/m2m_R/len": 256.0, + "TMR/t2m_gt_R/R01": 80.47, + "TMR/t2m_gt_R/R02": 88.28, + "TMR/t2m_gt_R/R03": 91.02, + "TMR/t2m_gt_R/R05": 92.58, + "TMR/t2m_gt_R/R10": 94.53, + "TMR/t2m_gt_R/MedR": 1.0, + "TMR/t2m_gt_R/len": 256.0, + "TMR/FID/gen_gt": 0.0387, + "TMR/FID/gt_text": 0.1349 + } +} +``` diff --git a/docs/source/benchmark/pipeline.md b/docs/source/benchmark/pipeline.md new file mode 100644 index 0000000000000000000000000000000000000000..779d0369d5016c15859f6cb45b752e04d133a71f --- /dev/null +++ b/docs/source/benchmark/pipeline.md @@ -0,0 +1,184 @@ +# Evaluation Pipeline + +This page describes the full benchmark workflow, which uses scripts in the `benchmark` directory: + +1. Build full test suite using ground-truth motions from BONES-SEED BVH data and benchmark metadata (`create_benchmark.py`), +2. Generate motions with a model for all or part of the test suite (`generate_eval.py`), +3. Compute text/motion embeddings with pre-trained TMR model (`embed_folder.py `), +4. Evaluate metrics over all generated samples (`evaluate_folder.py`), +5. Aggregate and summarize results (`parse_folder.py`). + +This pipeline works off-the-shelf for Kimodo models. To evaluate your own model, step (2) will need to be modified to generate with your custom model and output in the expected npz format. + +## Prerequisite: Download Motion Data and Metadata +The benchmark is constructed from motions in the BONES-SEED dataset and our released metadata. Make sure you have downloaded the [BONES-SEED dataset](https://huggingface.co/datasets/bones-studio/seed) along with the metadata for the test suite from HuggingFace at [`nvidia/Kimodo-Motion-Gen-Benchmark`](https://huggingface.co/datasets/nvidia/Kimodo-Motion-Gen-Benchmark). + +The `testsuite` folder from the downloaded metadata contains the directory structure described in the [benchmark introduction](introduction.md) with `meta.json`, `seed_motion.json`, and `seed_constraints.json` metadata files in the leaf folders. These metadata files contain info about the text prompts, durations, and constraint definitions for each test case. The first two steps of the evaluation pipeline will create the following in the leaf folders to prepare for computing metrics: + +- **Ground-Truth Motion** (`gt_motion.npz`): produced by `create_benchmark.py` from SEED BVH + metadata. +- **Constraints Configuration** (`constraints.json`): for test cases with constraint inputs, this file is created by `create_benchmark.py` from SEED BVH + metadata. +- **Generated Motion** (`motion.npz`): produced by the generation step from the model to evaluate (e.g. `generate_eval.py`). + +To perform the full evaluation, including metrics for both ground-truth and generated motions (steps 3--5), each leaf folder must contain both `gt_motion.npz` and `motion.npz`. + +> Note: all of the following steps will work with a _subset_ of the full test suite, if desired. Anywhere the `testsuite` directory is passed in, it can be replaced with a specific subset such as `testsuite/content/text2motion` to only run this subset of the benchmark. + +## 1. Build Full Benchmark (`create_benchmark.py`) + + The `create_benchmark.py` script bridges the ground truth motions and metadata: it downloads the testsuite structure (if not already present locally), then reads the referenced BVH files from a local copy of BONES-SEED and writes `gt_motion.npz` and `constraints.json` into each sample folder. + +```bash +python benchmark/create_benchmark.py path/to/testsuite --dataset datasets/bones-seed/soma_uniform +``` + +By default, this construction can take several hours and the resulting folder is about **26 GB**. + +To run faster, you can increase the number of parallel workers for processing: +```bash +OMP_NUM_THREADS=2 python benchmark/create_benchmark.py path/to/testsuite --dataset datasets/bones-seed/soma_uniform --workers 16 +``` +This example runs well with a 32-core system, but you may need to adjust the number of threads-per-worker and total workers for your system. Generally, a lower number of threads-per-worker with larger number of workers (up to your available CPU capacity) runs fastest. + +Options: + +- `--dataset`: path to the local SEED dataset folder (default: `datasets/bones-seed/soma_uniform`). +- `--workers`: number of parallel workers to use for benchmark construction (default: 1, sequential) +- `--overwrite`: rebuild `gt_motion.npz` even if it already exists. + +For each test case, the script: + +1. parses the BVH file into local rotation matrices and root translation, +2. subsamples to 30 FPS, +3. converts to the standard T-pose via `SOMASkeleton77.to_standard_tpose`, +4. computes Kimodo motion features and canonicalizes the motion, +5. writes the resulting motion dictionary as `gt_motion.npz`. + +For a detailed walkthrough of steps 1--4, see [Loading BONES-SEED BVH data](../user_guide/seed_dataset.md). + +## 2. Generate Motions (`generate_eval.py`) + +The next step is to generate a motion for each test case. +The script `benchmark/generate_eval.py` recursively generates one motion with Kimodo per test case from either the full `testsuite` or a desired subset. + +```bash +python benchmark/generate_eval.py \ + --benchmark path/to/testsuite \ + --output generated_folder \ + --model kimodo-soma-rp \ + --batch_size 32 \ + --num_workers 4 +``` + +The batch size and number of data workers should be adjusted for your system. The script is intended to be run with the latest Kimodo-SOMA models (right now v1.1) which are compatible with the benchmark. + +> Note: each test cases has a seed in `meta.json` that is loaded and used for generation to enable reproducibility. However, by default, the generation script uses the first seed in a batch to seed the whole batch, so to make results completely repeatable, you must set the batch size to 1 or always use the same batch size when running generation. + +Useful options: + +- `--model`: Kimodo model to use for generation. See [available models](../getting_started/quick_start.md#overview-kimodo-models) for the full list. +- `--output`: output root directory. The testsuite hierarchy is mirrored here. If omitted, motions are generated **in-place** inside the testsuite folder. +- `--overwrite`: regenerate even if `motion.npz` already exists. +- `--diffusion_steps`: default denoising steps (can be overridden by each sample `meta.json`). +- `--postprocess`: enable post-processing. For fair evaluation, it is recommended to **not** use post-processing so that metrics reflect the raw model output. +- `--text_encoder_fp32`: will instantiate the text encoder (if needed) with float32 precision instead of bfloat16. The Kimodo v1.1 models are trained with float32 text encodings, so this slightly improves accuracy but requires extra VRAM. + +After generation, the output tree mirrors the `testsuite` hierarchy and includes generated motions (`motion.npz`). If the testsuite was built with `create_benchmark.py`, each leaf already has `gt_motion.npz`; the generation step adds `motion.npz` per sample. + +```text +generated_folder/ +└── .../0000/ + ├── meta.json + ├── constraints.json # present if available in testsuite + ├── gt_motion.npz # if built with create_benchmark + └── motion.npz # generated +``` + +### Using Custom Models + +The `generate_eval` script is set up to work with Kimodo models, but it can be easily adapted or replaced by generation with a custom model. The only requirement to be able to compute all metrics is to output the `motion.npz` file for each test case that minimally contains: (1) `posed_joints` field with global joint positions on the SOMA 77-joint skeleton and (2) `foot_contacts` field with binary foot contact predictions. Please see the [output formats docs](../user_guide/output_formats.md) for more details on the `NPZ` format. + +## 3. Embed with Pre-Trained TMR (`embed_folder.py`) + +Several evaluation metrics such as R-precision, FID, and latent similarity rely on latent embeddings of both motion and text. For this purpose, we use a [Text-Motion-Retrieval (TMR)](https://mathis.petrovich.fr/tmr/) model trained on the full Bones Rigplay dataset. See [Metrics](metrics.md) for details on the TMR evaluation protocol and metrics. + +The next step in the eval pipeline is using this TMR model with the `benchmark/embed_folder.py` script to recursively embed each generated motion (`motion.npz`), GT motion (`gt_motion.npz`) when present, and the text prompt from `meta.json`: + +```bash +python benchmark/embed_folder.py generated_folder --model tmr-soma-rp +``` + +The default TMR model (`tmr-soma-rp`) trained on the full Rigplay dataset is released as [`TMR-SOMA-RP-v1`](https://huggingface.co/nvidia/TMR-SOMA-RP-v1). It is automatically downloaded from HuggingFace on first use of the embedding script. + +Options: + +- `--model`: TMR model to use for encoding (default: `tmr-soma-rp`). +- `--device`: compute device (`cuda` or `cpu`). Defaults to `cuda` if available, otherwise `cpu`. +- `--overwrite`: re-embed even if embedding files already exist. +- `--text_encoder_fp32`: will instantiate the text encoder (if needed) with float32 precision instead of bfloat16. The TMR model is trained with float32 text encodings, so this slightly improves accuracy but requires extra VRAM. + +Running this script saves the embeddings to each test case folder that has the corresponding motion file(s) and `meta.json`: + +- `motion_embedding.npy` (when `motion.npz` exists) +- `gt_motion_embedding.npy` (when `gt_motion.npz` exists) +- `text_embedding.npy` + +> Note: this script can take over 1 hour to run for the full test suite, depending on your GPU. + +## 4. Compute Evaluation Metrics (`evaluate_folder.py`) + +Next, use `benchmark/evaluate_folder.py` to compute per-test-case and aggregated metrics across the test suite (or a specific subset folder). Each leaf folder must contain both `motion.npz` and `gt_motion.npz` to compute the metrics. + +```bash +python benchmark/evaluate_folder.py generated_folder +``` + +Options: + +- `--device`: compute device (`cuda` or `cpu`). Defaults to `cuda` if available, otherwise `cpu`. + +The script runs two evaluation passes: one on the generated motion (`motion.npz`) and one on the ground-truth motion (`gt_motion.npz`). It outputs: + +- per test case results: `metrics.json` inside each test case (leaf) folder with metrics summarized for that single test case +- per group results: `.json` one level above each group of test-case folders that aggregates metrics over all contained test cases + +Please see the [Metrics](metrics.md) page for a detailed explanation of these json formats. + +After embedding and evaluation, the folder structure should look like: + +```text +generated_folder/ +├── .../0000/ +│ ├── motion.npz +│ ├── gt_motion.npz +│ ├── motion_embedding.npy +│ ├── gt_motion_embedding.npy +│ ├── text_embedding.npy +│ └── metrics.json # single test-case metrics +└── .../.json # folder-level aggregate summary of all contained test cases +``` + +## 5. Summarize Results of Full Benchmark (`parse_folder.py`) + +If you have computed metrics for the _entire_ test suite (both `content` and `repetition` splits), use `benchmark/parse_folder.py` to validate all per-test-case result JSONs and aggregate metrics into summary tables. Unlike the previous steps, this script expects the user to pass in the root `testsuite` and for the test suite to follow the standard split/category hierarchy (see [Introduction](introduction.md)): + +- **Splits**: `content`, `repetition` +- **Categories**: `overview`, `timeline_single`, `timeline_multi` (text-following), `constraints_withtext`, `constraints_notext` (constrained generation) + +```bash +python benchmark/parse_folder.py generated_folder +``` + +Options: + +- `--output`: path for the output JSON (default: `/summary_rows.json`). +- `--format`: table output format. `terminal` (default) for fixed-width tables, `md` for markdown tables suitable for copy-pasting into documentation. + +The script: + +1. discovers all grouped test case directories (folders containing single test cases with `meta.json`, `motion.npz`, and `gt_motion.npz`), +2. loads each group's `.json` result files written by `evaluate_folder`, +3. computes weighted averages of all metrics by split and category, +4. writes `summary_rows.json` with per-row and per-table aggregated results, +5. prints formatted benchmark tables to the terminal (text-following and constraints, with GT and method rows side by side). + +Metric values in the tables are converted to user-friendly units (e.g., constraint position errors in cm, foot skating in cm/s). See [Metrics](metrics.md) for definitions of individual metrics. diff --git a/docs/source/benchmark/results.md b/docs/source/benchmark/results.md new file mode 100644 index 0000000000000000000000000000000000000000..ec947c31e79c28adef4d4f5d210b631a2572ba9e --- /dev/null +++ b/docs/source/benchmark/results.md @@ -0,0 +1,97 @@ +# Kimodo Results + +On this page, we report the results for the latest Kimodo models on the benchmark test suite. These results are reproducible with the [evaluation pipeline](pipeline.md) and should be used when comparing against other models. Note that the reported numbers differ from the numbers in the [tech report](https://research.nvidia.com/labs/sil/projects/kimodo/assets/kimodo_tech_report.pdf) (Sec. 6) due to differences in skeleton, test suite composition, and evaluation details. + +To reproduce these results or evaluate your own model, follow the [evaluation pipeline](pipeline.md) and use `parse_folder --format md` to generate summary tables in markdown format. + +**Note on reproducibility**: to exactly reproduce the results in the tables below, use batch size 1 when generating with Kimodo (i.e., when running `generate_eval.py`). This way, every test case is individually seeded according to `meta.json`. The reported results were computed using LLM2Vec in the default `bfloat16` precision. However, the Kimodo-SOMA-v1.1 and TMR models were actually trained with `float32` embeddings, so if you want to get the best possible performance (and you have enough VRAM), you can include `--text_encoder_fp32` when running the generation and embedding steps, even though the results will not match the tables here. + +Results are reported on the two splits described in [the introduction](introduction.md#dataset-splits): + +- **Content**: test cases with novel semantic content not present in training (e.g. unseen action categories). +- **Repetition**: content categories seen during training, but specific motion clips are held out and unseen. Note that due to the annotations in Bones Rigplay and SEED datasets, the text prompts in this test split have already been seen during training. + +For each split, we also report metrics for the ground truth motion. These rows serve as an empirical upper bound for motion quality, and deviations between ground truth and generated metrics highlight where the model can improve. + +We split results for each model into two tables corresponding to different test cases in the test suite: + +- **Text-Following**: `overview`, `timeline_single`, and `timeline_multi` +- **Constrained**: `constraints_withtext`, `constraints_notext` + + + + +## Quantitative Results + +Results are reported for two models: + +- **Kimodo-SOMA-SEED-v1.1**: trained on the public [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) dataset. The results are comparable to any model trained on SEED that uses our recommended splits [described in the introduction](introduction.md#dataset-splits). +- **Kimodo-SOMA-RP-v1.1**: trained on the full (proprietary) Bones Rigplay dataset which is a superset of BONES-SEED. Though the training split is larger, the model is not trained on the SEED test splits to ensure a fair comparison. + +### Text-Following Evaluation + +| | Overview R@3↑ | Overview FID↓ | Overview Skate↓ | Overview Contact↑ | Timeline single R@3↑ | Timeline single FID↓ | Timeline single Skate↓ | Timeline single Contact↑ | Timeline multi R@3↑ | Timeline multi FID↓ | Timeline multi Skate↓ | Timeline multi Contact↑ | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| **Content** Ground Truth | 89.09 | 0.000 | 1.849 | 1.000 | 86.26 | 0.000 | 1.789 | 1.000 | 88.47 | 0.000 | 1.711 | 1.000 | +| **Content** Kimodo-SOMA-SEED-v1.1 | 81.13 | 0.035 | 4.077 | 0.977 | 73.17 | 0.028 | 3.873 | 0.980 | 80.10 | 0.032 | 3.685 | 0.981 | +| **Content** Kimodo-SOMA-RP-v1.1 | 83.32 | 0.025 | 3.641 | 0.982 | 78.08 | 0.026 | 3.523 | 0.984 | 84.79 | 0.028 | 3.278 | 0.985 | +| **Repetition** Ground Truth | 93.91 | 0.000 | 2.106 | 1.000 | 90.13 | 0.000 | 2.037 | 1.000 | 94.49 | 0.000 | 1.931 | 1.000 | +| **Repetition** Kimodo-SOMA-SEED-v1.1 | 90.92 | 0.004 | 4.573 | 0.972 | 80.38 | 0.007 | 4.442 | 0.976 | 92.58 | 0.006 | 4.199 | 0.974 | +| **Repetition** Kimodo-SOMA-RP-v1.1 | 87.90 | 0.008 | 4.103 | 0.977 | 77.02 | 0.011 | 3.938 | 0.981 | 88.59 | 0.009 | 3.727 | 0.980 | + +### Constrained Evaluation + +| | With text FB Pos↓ | With text EE Pos↓ | With text EE Rot↓ | With text 2D Root↓ | With text Pelvis@95% | Without text FB Pos↓ | Without text EE Pos↓ | Without text EE Rot↓ | Without text 2D Root↓ | Without text Pelvis@95% | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| **Content** Ground Truth | 0.000 | 0.000 | - | 3.837 | 5.36 | 0.000 | 0.000 | - | 3.913 | 5.41 | +| **Content** Kimodo-SOMA-SEED-v1.1 | 3.421 | 3.817 | - | 4.979 | 9.14 | 3.320 | 3.664 | - | 4.797 | 9.03 | +| **Content** Kimodo-SOMA-RP-v1.1 | 2.929 | 3.029 | - | 4.581 | 7.77 | 2.935 | 2.994 | - | 4.411 | 7.37 | +| **Repetition** Ground Truth | 0.000 | 0.000 | - | 3.607 | 5.44 | 0.000 | 0.000 | - | 3.567 | 5.42 | +| **Repetition** Kimodo-SOMA-SEED-v1.1 | 3.187 | 3.852 | - | 4.734 | 9.19 | 3.120 | 3.510 | - | 4.264 | 7.89 | +| **Repetition** Kimodo-SOMA-RP-v1.1 | 2.804 | 2.983 | - | 4.260 | 7.63 | 2.829 | 2.969 | - | 4.027 | 7.21 | \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000000000000000000000000000000000000..5676d59aa38c4de85b5835e7bb01395ad290ba08 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +# -- Path setup -------------------------------------------------------------- +sys.path.insert(0, os.path.abspath("../..")) + +# -- Project information ----------------------------------------------------- + +project = "Kimodo" +copyright = "2026, NVIDIA" +author = "NVIDIA" + +version = "" +release = "" + +# -- General configuration --------------------------------------------------- + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx.ext.intersphinx", + "sphinx.ext.autosummary", + "sphinx.ext.githubpages", + "sphinx_copybutton", + "myst_parser", + "sphinx_design", +] + +napoleon_google_docstring = True +napoleon_numpy_docstring = False +napoleon_include_init_with_doc = True +napoleon_use_param = True +napoleon_use_rtype = True + +autodoc_default_options = { + "members": True, + "member-order": "bysource", + "special-members": "__init__", + "undoc-members": True, + "exclude-members": "__weakref__", + "show-inheritance": False, +} +autodoc_typehints = "none" + +autosummary_generate = True + +# Avoid initialization issues for optional native libs +os.environ.setdefault("MUJOCO_GL", "osmesa") +os.environ.setdefault("PYOPENGL_PLATFORM", "osmesa") + + +class Mock: + """Mock class for imports that can't be satisfied.""" + + def __init__(self, *args, **kwargs): + pass + + def __call__(self, *args, **kwargs): + return Mock() + + def __getattr__(self, name): + if name in ("__file__", "__path__"): + return "/dev/null" + if name == "__version__": + # Some libraries (e.g. safetensors) parse torch.__version__ with + # packaging.version.Version, so this must be a valid PEP 440 string. + return "0.0.0" + if name == "__signature__": + return None + if name == "__mro_entries__": + return lambda bases: () + return Mock() + + def __getitem__(self, name): + return Mock() + + def __iter__(self): + return iter([]) + + def __or__(self, other): + return Mock() + + def __ror__(self, other): + return Mock() + + +mock_modules = [ + "torch", + "torch.nn", + "torch.nn.functional", + "torch.optim", + "torch.distributed", + "torch.cuda", + "torch.utils", + "torch.utils.data", + "lightning", + "lightning.fabric", + "lightning_fabric", + "pytorch_lightning", + "tensordict", + "pydantic", + "pydantic.dataclasses", + "pydantic_core", + "mujoco", + "isaacgym", + "isaacgymenvs", + "genesis", + "omni", + "wandb", + "hydra", + "omegaconf", + "tqdm", + "trimesh", + "pyvista", + "smplx", + "smpl", + "scipy", + "scipy.spatial", + "scipy.spatial.transform", + "peft", + "transformers", + "safetensors", + "safetensors.torch", + "sklearn", + "PIL", + "cv2", + "rich", + "rich.progress", + "skimage", + "imageio", + "openmesh", + "gym", + "easydict", + "dm_control", + "dm_control.mjcf", + "dm_control.mujoco", + "matplotlib", + "matplotlib.pyplot", +] + +for mod in mock_modules: + sys.modules[mod] = Mock() + +autodoc_mock_imports = mock_modules + +templates_path = ["_templates"] +exclude_patterns = ["api_reference/_generated/**"] + +language = "en" + +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} + +master_doc = "index" + +# -- Options for HTML output ------------------------------------------------- + +html_theme = "nvidia_sphinx_theme" +html_static_path = ["_static"] +html_css_files = ["custom.css"] +html_logo = "_static/logo-placeholder.svg" +html_show_sourcelink = False + +html_theme_options = { + "collapse_navigation": False, + "navigation_depth": 4, +} + +toc_object_entries_show_parents = "hide" + +htmlhelp_basename = "Kimododoc" + +# -- Options for intersphinx ------------------------------------------------- + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "torch": ("https://pytorch.org/docs/stable/", None), + "numpy": ("https://numpy.org/doc/stable/", None), +} + +copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " +copybutton_prompt_is_regexp = True + +# Generate heading anchors so cross-doc links like path.md#fragment resolve (local ids). +myst_heading_anchors = 4 + +# Required so `:::{dropdown}` and other fenced directives in .md files are parsed (not shown as plain text). +myst_enable_extensions = ["colon_fence"] + + +def setup(app): + app.add_css_file("custom.css") diff --git a/docs/source/getting_started/installation.md b/docs/source/getting_started/installation.md new file mode 100644 index 0000000000000000000000000000000000000000..ab6a961dd147f41fe7fe38edcaef7f18349a2b83 --- /dev/null +++ b/docs/source/getting_started/installation.md @@ -0,0 +1,69 @@ +# Installation + +> Note: This project will download and install additional third-party open source software projects. Review the license terms of these open source projects before use. + +> Note: This repo was developed and primarily tested on Linux + +There are two ways to install Kimodo: (1) as a package, or (2) download the source code and install. +Both require setting up a Hugging Face token to use the text encoder at generation time. + +## Set Up Hugging Face Token + +The Kimodo text encoder relies on the **gated** `meta-llama/Meta-Llama-3-8B-Instruct` model, which requires: +- Your HF account has been granted access to the [model page](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct). +- You provide a HF token for runtime + +After receiving access to the Llama repo, please create an access token [here](https://huggingface.co/settings/tokens/new?tokenType=read). Then use it to log in on your command line: + +```bash +hf auth login +``` +or alternatively, paste the token in this file ``~/.cache/huggingface/token``. If you don't have `hf` installed, you will first need to run `pip install --upgrade huggingface_hub`. + +## Kimodo Install Option 1: Package Install + +The easiest way to get started is simply installing Kimodo as a package without needing to clone the codebase. This will allow you to generate motions and run the demo as a black box. + +We suggest creating a new Python environment for the install, for example with `venv` or conda: +```bash +conda create -n kimodo python=3.10 +conda activate kimodo +``` + +To ensure you have a version of [PyTorch](https://pytorch.org/get-started/locally/) that is compatible with your system and CUDA version, it is recommended to manually install the best version of PyTorch for you before installing Kimodo. Anything over PyTorch 2.0 is sufficient. We strongly suggest using a GPU-capable version of PyTorch to generate motions in a reasonable amount of time. + +Installing the base Kimodo package will allow you generate motions with the command line: +```bash +pip install git+https://github.com/nv-tlabs/kimodo.git +``` + +If you want to be able to run the interactive demo as well, use this command which installs additional dependencies: +```bash +pip install "kimodo[all] @ git+https://github.com/nv-tlabs/kimodo.git" +``` + +Now should be ready to use Kimodo. Check out the [quick start guide](quick_start.md) to see how to generate motions. + +If you experience issues with package or system compatibility using the above install strategy, we recommend downloading the codebase and using the Docker install detailed below. + +## Kimodo Install Option 2: Source Code Install + +If you plan to build on Kimodo or dig into the codebase, you'll want to clone and install the repo. + +### Clone Kimodo Repository + +```bash +git clone https://github.com/nv-tlabs/kimodo.git +cd kimodo +``` + +### Choose Your Installation Route +Kimodo can be installed by building and running through a virtual environment (e.g., `conda`) or within a Docker container. + +```{toctree} +:maxdepth: 1 + +installation_virtual_env +installation_docker +installation_smpl +``` diff --git a/docs/source/getting_started/installation_docker.md b/docs/source/getting_started/installation_docker.md new file mode 100644 index 0000000000000000000000000000000000000000..6f2f54dc2d79364024ebf9a441e00ee7fac69da8 --- /dev/null +++ b/docs/source/getting_started/installation_docker.md @@ -0,0 +1,60 @@ +# Installation With Docker + +> Note: the first time building and running with Docker can take several minutes, please be patient. + +## Clone Modified Viser Library +The interactive demo relies on [a fork of Viser](https://github.com/nv-tlabs/kimodo-viser) that implements a timeline interface and more. Clone it within the `kimodo` directory before building with Docker using: +```bash +git clone https://github.com/nv-tlabs/kimodo-viser.git +``` + +## Quick Install + +Before running Docker, make sure your Hugging Face token is available at +`~/.cache/huggingface/token` on the host, for example by running +`hf auth login` once outside the container (see the [Installation](installation.md) instructions). + +The easiest way to build and immediately run the interactive demo webapp (with the text-encoder service) in one command is: + +```bash +docker compose up -d --build +``` + +## Step-by-Step Installation + +Alternatively, you can first build with: + +```bash +docker compose build +``` + +This builds `text-encoder` and `demo` containers corresponding to the text encoding service and the interactive motion authoring webapp, respectively. Please see the [quick start guide](quick_start.md) for more information on these. + +
+ +Advanced Configuration of Dependencies + +This repo uses: +- `docker_requirements.in`: human-maintained, top-level dependencies +- `docker_requirements.txt`: pinned lockfile (automatically generated) + +Notes: +- We keep a lockfile for **reproducible Docker builds** (so a rebuild next week pulls the same deps). +- The lockfile intentionally **omits `torch`/CUDA wheels** because the Docker base image + (`nvcr.io/nvidia/pytorch`) already provides a tested PyTorch build (avoids slow installs and CUDA mismatches). + +
+
+ +After building, you will need to manually start the text-encoder service before doing any motion generation: +```bash +docker compose up text-encoder +``` +Note, the first time running this command will take a long time as the Llama-based text encoder is downloaded. + +Finally, to start the interactive demo: +```bash +docker compose up demo +``` + +For more information on using the Docker setup, see the [Quick Start](quick_start.md) guide next. diff --git a/docs/source/getting_started/installation_smpl.md b/docs/source/getting_started/installation_smpl.md new file mode 100644 index 0000000000000000000000000000000000000000..7ef0424017694803fc8a4f63bd618e1a25e760e2 --- /dev/null +++ b/docs/source/getting_started/installation_smpl.md @@ -0,0 +1,17 @@ +# Using Kimodo-SMPLX Model + +Using the [Kimodo-SMPLX-RP-v1](https://huggingface.co/nvidia/Kimodo-SMPLX-RP-v1) model requires a few extra installation steps. + +## Request Model Access + +The SMPL-X version of Kimodo is gated, so before trying to generate motions with it in the CLI or demo, go to the [Hugging Face model page](https://huggingface.co/nvidia/Kimodo-SMPLX-RP-v1) and request access. As described in the [installation](./installation.md) process, make sure your HF token is properly set up so your access to the model can be authenticated. + +## Download SMPL-X Body Model +If you want to visualize generated SMPL-X motions in the demo, you will need to download the SMPL-X body model. +Go to the [SMPL-X](https://smpl-x.is.tue.mpg.de/) webpage and then sign in or create an account and go to the "Download" page. +Click "Download SMPL-X with removed head bun (NPZ)" and then copy the `SMPLX_NEUTRAL.npz` file to the Kimodo codebase to be at `kimodo/kimodo/assets/skeletons/smplx22/SMPLX_NEUTRAL.npz`. + +Note that if you installed Kimodo as a package without downloading the codebase, you'll need to find where the assets directory is located by running: +```bash +python -c "from kimodo.assets import skeleton_asset_path; print(skeleton_asset_path('smplx22'))" +``` diff --git a/docs/source/getting_started/installation_virtual_env.md b/docs/source/getting_started/installation_virtual_env.md new file mode 100644 index 0000000000000000000000000000000000000000..7045a414b0a82d38769436af1c221deb7d6ce5c8 --- /dev/null +++ b/docs/source/getting_started/installation_virtual_env.md @@ -0,0 +1,45 @@ +# Installation With Virtual Environment + +> Note: the repo was tested with Python 3.10+ and PyTorch 2.0+. + +## Create Enviroment +We recommend setting up a separate virtual environment for Kimodo to avoid dependency conflicts. + +### Using venv +```bash +python -m venv venv +source venv/bin/activate +``` + +### Using Conda +```bash +conda create -n kimodo python=3.10 +conda activate kimodo +``` + +## Install Dependencies + +### Install PyTorch +First, make sure to install a version of [PyTorch](https://pytorch.org/get-started/locally/) that works with your system and CUDA version. We suggest anything over PyTorch 2.0. We strongly suggest using a GPU-capable version of PyTorch to generate motions in a reasonable amount of time. + +### (Optional) Clone Modified Viser Library +The interactive demo relies on [a fork of Viser](https://github.com/nv-tlabs/kimodo-viser) that implements a timeline interface and more. If you want to have an editable install of this version of Viser (i.e., you expect to modify it), clone and install it within the `kimodo` directory using: +```bash +git clone https://github.com/nv-tlabs/kimodo-viser.git +pip install -e kimodo-viser +``` + +### Install Kimodo +Next, install Kimodo run this command from the base of repo: +```bash +pip install -e . +``` +This results in a single editable install for Kimodo and the MotionCorrection package. + +If you plan to use the demo, you can instead run: +```bash +pip install -e ".[all]" +``` +This will install our [Viser fork](https://github.com/nv-tlabs/kimodo-viser) (if not already installed in the previous step) and the [SOMA body model](https://github.com/NVlabs/SOMA-X). + +Next, head over to the [Quick Start](quick_start.md) page to test out your installation by generating some motions. diff --git a/docs/source/getting_started/quick_start.md b/docs/source/getting_started/quick_start.md new file mode 100644 index 0000000000000000000000000000000000000000..29897c9d2af8da13277047cb1fc1ac94fbf4fef0 --- /dev/null +++ b/docs/source/getting_started/quick_start.md @@ -0,0 +1,119 @@ +# Quick Start + +This page provides a quick introduction to motion generation with Kimodo. For detailed explanations, we recommend reviewing the full documentation pages linked in each section. + +Before running these commands, follow the [installation guide](installation.md) to install Kimodo in a virtual environment or using Docker. + +## Overview: Kimodo Models +Motion generation can be performed with several trained Kimodo models that vary by skeleton and training dataset. + +> Note: models will be downloaded automatically when attempting to generate from the CLI or Interactive Demo, so there is no need to download them manually + +| Model | Skeleton | Training Data | Release Date | Hugging Face | License | +|-------|------|------|-------------|-------------|----| +| **Kimodo-SOMA-RP-v1.1** | [SOMA](https://github.com/NVlabs/SOMA-X) | [Bones Rigplay 1](https://bones.studio/datasets#rp01) | April 10, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-SOMA-RP-v1.1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-SOMA-SEED-v1.1** | [SOMA](https://github.com/NVlabs/SOMA-X) | [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) | April 10, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-SOMA-SEED-v1.1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-SOMA-RP-v1** | [SOMA](https://github.com/NVlabs/SOMA-X) | [Bones Rigplay 1](https://bones.studio/datasets#rp01) | March 16, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-SOMA-RP-v1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-G1-RP-v1** | [Unitree G1](https://github.com/unitreerobotics/unitree_mujoco/tree/main/unitree_robots/g1) | [Bones Rigplay 1](https://bones.studio/datasets#rp01) | March 16, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-G1-RP-v1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-SOMA-SEED-v1** | [SOMA](https://github.com/NVlabs/SOMA-X) | [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) | March 16, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-SOMA-SEED-v1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-G1-SEED-v1** | [Unitree G1](https://github.com/unitreerobotics/unitree_mujoco/tree/main/unitree_robots/g1) | [BONES-SEED](https://huggingface.co/datasets/bones-studio/seed) | March 16, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-G1-SEED-v1) | [NVIDIA Open Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-open-model-license/) | +| **Kimodo-SMPLX-RP-v1** | [SMPL-X](https://github.com/vchoutas/smplx) | [Bones Rigplay 1](https://bones.studio/datasets#rp01) | March 16, 2026 | [Link](https://huggingface.co/nvidia/Kimodo-SMPLX-RP-v1) | [NVIDIA R&D Model](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-internal-scientific-research-and-development-model-license/) | + +By default, we recommend using the models trained on the full Bones Rigplay dataset (700 hours of mocap) for your motion generation needs. +The models trained on BONES-SEED use 288 hours of [publicly available mocap data](https://huggingface.co/datasets/bones-studio/seed) so are less capable, but are useful for comparing your own trained models on the same dataset. See the [benchmark](../benchmark/introduction.md) for a standardized evaluation suite on BONES-SEED. + +### Recommended Hardware +Kimodo requires ~17GB of VRAM to generate locally entirely on GPU, due primarily to the size of the text embedding model. If you have a smaller card, set `TEXT_ENCODER_DEVICE=cpu` when running Kimodo commands to force text encoding to the CPU. This is slightly slower but reduces VRAM usage to <3 GB. + +The model has been most extensively tested on GeForce RTX 3090, GeForce RTX 4090, and NVIDIA A100 GPUs, but it should work on other recent cards with sufficient VRAM. + +## Run Text-Encoder Service +Motion generation relies on embedding the input text prompt, which becomes the input to Kimodo. Although it is fine to run the CLI commands and demo on their own, it may be preferred to start the _text encoder service_ in the background, which can be shared across all motion generation requests. This is much more efficient when making many consecutive CLI calls, as it avoids needing to instantiate the large text encoder every time. + +To start the text encoder service: +```bash +kimodo_textencoder +``` + +The first run of the service will take a while as it downloads the embedding model. We recommend running this in the background or in a separate terminal where it will stay open and usable by other scripts. + +If you are using the Docker set up, the service can alternatively be started in the container with: +```bash +docker compose up text-encoder +``` + +> Note: when the text encoder is initialized, the transformers library will report several unexpected and missing layers for LLM2Vec. These are expected and can be safely ignored. + +If you are running on a GPU with <16 GB VRAM, you can force the text encoder to the CPU, for example: +```bash +TEXT_ENCODER_DEVICE=cpu kimodo_textencoder +``` + +## Command-Line Text-to-Motion Generation +**[CLI Documentation](../user_guide/cli.md)** + +You can generate motions from the command line using the generate script: + +```bash +kimodo_gen "A person walks forward." \ + --model Kimodo-SOMA-RP-v1 \ + --duration 5.0 \ + --output output +``` + +The `--model` command corresponds to the model name in the table above. The output motion will be saved using the stem name given by `--output` in the Kimodo [output format](../user_guide/output_formats.md). For a detailed description of all generation arguments, including how to generate motion with constraints, see the full [CLI documentation](../user_guide/cli.md). + +If you set up Kimodo with Docker, you can instead run generation inside the Docker container, replacing `kimodo_gen XXX` with `docker compose run --rm demo kimodo_gen XXX`. If you will be running generation multiple times, it is better to start the `demo` container (e.g., in another terminal or in the background), and then run commands inside it with `docker compose exec demo kimodo_gen XXX`. + + +## Interactive Motion Authoring Demo +**[Demo Documentation](../interactive_demo/index.md)** + +The demo allows easily generating motions with an intuitive control interface for text prompting and constraints. + +The demo can be started with: +```bash +kimodo_demo +``` + +The demo is a webapp that will run on [http://localhost:7860](http://localhost:7860). Open this URL in your browser to access the interface. + +If you are using Docker, the demo can be launched with: +```bash +docker compose up demo +``` +or if you want to start the demo and text encoder service (explained below) at the same time, use: +```bash +docker compose up +``` + +
+Additional Tips for Docker + +You may find the following commands useful if running Kimodo within the Docker containers. In the example commands below, you can also replace `demo` by `text-encoder`: + +**Check logs:** + +```bash +docker compose logs demo +``` + +**Stop service:** + +```bash +docker compose stop demo +``` + +**Restart service:** + +```bash +docker compose restart demo +``` + +**Stop and remove everything:** + +```bash +docker compose down +``` + +
diff --git a/docs/source/index.md b/docs/source/index.md new file mode 100644 index 0000000000000000000000000000000000000000..cdbc6fd7db921cdfbd2726c292e864d7ce3cc940 --- /dev/null +++ b/docs/source/index.md @@ -0,0 +1,100 @@ +# Kimodo Documentation + +
+
Kimodo
+
+ Scaling controllable human motion generation +
+
+ Get Started + Interactive Demo + GitHub + Tech Report +
+
+ + +## Overview + +Kimodo is a **ki**nematic **mo**tion **d**iffusi**o**n model trained on a large-scale (700 hours) commercially-friendly optical motion capture dataset. The model generates high-quality 3D human and robot motions, and is controlled through text prompts and an extensive set of constraints such as full-body pose keyframes, end-effector positions/rotations, 2D paths, and 2D waypoints. See the [project page](https://research.nvidia.com/labs/sil/projects/kimodo/) for details. + +## Highlights + +
+
+

Controlled Generation

+

Text prompts combined with full-body, root, and end-effector constraints.

+
+
+

Human(oid) Support

+

Model variations for both digital humans and humanoid robots.

+
+
+

Interactive Demo

+

Timeline editing, real-time 3D visualization, and example presets.

+
+
+ +## Quick links + +- [Installation](getting_started/installation.md) +- [Quick Start](getting_started/quick_start.md) +- [Command Line Interface](user_guide/cli.md) +- [Interactive Demo](interactive_demo/index.md) +- [Project Structure](project_structure.md) + +```{toctree} +:maxdepth: 3 +:caption: Getting Started +:hidden: + +getting_started/installation +getting_started/quick_start +``` + +```{toctree} +:maxdepth: 2 +:caption: User Guide +:hidden: + +interactive_demo/index +user_guide/cli +user_guide/constraints +user_guide/output_formats +user_guide/motion_convert +user_guide/seed_dataset +user_guide/configuration +``` + +```{toctree} +:maxdepth: 2 +:caption: Key Concepts +:hidden: + +key_concepts/model +key_concepts/limitations +key_concepts/motion_representation +key_concepts/constraints +key_concepts/skeleton +``` + +```{toctree} +:maxdepth: 2 +:caption: Benchmark +:hidden: + +benchmark/introduction +benchmark/pipeline +benchmark/metrics +benchmark/results +``` + +```{toctree} +:maxdepth: 2 +:caption: Reference +:hidden: + +project_structure +project_info +api_reference/index +``` diff --git a/docs/source/interactive_demo/constraints.md b/docs/source/interactive_demo/constraints.md new file mode 100644 index 0000000000000000000000000000000000000000..3c570fbc3e8ff616bb8c4b14b76777a2cb2a350c --- /dev/null +++ b/docs/source/interactive_demo/constraints.md @@ -0,0 +1,15 @@ +# Constraints + +Constraints guide the motion at specific frames or intervals. To learn about the types of constraints details of each, see the [constraints concepts](../key_concepts/constraints.md) and [constraints format](../user_guide/constraints.md) pages. + +![Constraints panel](../_static/demo/constraints_panel.png) +![Editing mode](../_static/demo/editing_mode.png) + +The constraint panel allows you to configure constraints and editing: + +- **Enter Editing Mode**: enable FK pose editing in the viewer. Gizmos will be displayed on joints that can be edited. If there is already a constraint on the timeline for the current frame, any pose editing will adjust that constraint, otherwise you need to add a constraint on the timeline after adjusting the pose. +- **Gizmo space**: whether to display the rotation gizmos in local or global joint space while editing +- **Snap to Constraint**: will snap the current frame of motion to the constraint at that frame. This can be useful if a generated pose does not exactly meet the constraint and you want to continue editing the constraint. +- **Reset Constraint**: does the opposite by snapping the pose back to the original generated motion from the constrained pose. +- **Root 2D Options > Make Smooth Path**: if you have laid down root waypoint constraints, checking this box will turn the waypoints into a smoothed dense path constraint. If there is not a waypoint at the first and last frames of the motion, they will be automatically added since Kimodo is only trained on full-sequence paths. +- **Clear All Constraints**: clears all current constraints from the viewer and timeline. diff --git a/docs/source/interactive_demo/examples.md b/docs/source/interactive_demo/examples.md new file mode 100644 index 0000000000000000000000000000000000000000..d3ad1e18087e8409a32b5bbbf4d356bd8f07994a --- /dev/null +++ b/docs/source/interactive_demo/examples.md @@ -0,0 +1,17 @@ +# Examples + +The Examples Tab within the settings panel contains several examples that highlight the key capabilities and potential workflows with Kimodo. +Examples are included for the `Kimodo-SOMA-RP` and `Kimodo-G1-RP` models. + +![Skeleton overview](../_static/demo/examples_panel.png) + +After choosing an example from the dropdown menu, click "Load Example" to load the example configuration into the viewer. + +The viewer will display the pre-generated motion along with the prompts and constraints on the timeline that were used to generate it. All settings used to generate the model are also loaded with the example (e.g., seed, classifier-free guidance settings), so you should be able to click "Generate" in the panel to recover the same result. + +Example cover a variety of ways to use one or more text prompts along with kinematic constraints for generation. + +**Saving New Examples**: after you've generated a motion, you can save a new Example under the "Load/Save" tab of the Settings panel. You should immediately see the Examples dropdown update with your new saved example so it can be loaded in later. + +This section walks through common workflows and how to use the webapp. Each +workflow has its own section and an accompanying video. diff --git a/docs/source/interactive_demo/export_results.md b/docs/source/interactive_demo/export_results.md new file mode 100644 index 0000000000000000000000000000000000000000..ff5e1bea4dcab43b9b03eb9a611e7b6d2876088e --- /dev/null +++ b/docs/source/interactive_demo/export_results.md @@ -0,0 +1,19 @@ +# Saving/Loading + +The Load/Save and Exports panels allow saving generated results and load in previously generated results + +![Export panel](../_static/demo/exports_panel.png) + +- **Load/Save** + - **Motion**: save the current motion in the [NPZ format](../user_guide/output_formats.md#kimodo-npz-format) to a specific path. Motion NPZs can also be loaded into the viewer from this panel. This is useful to load in motions generated with the CLI. + - **Constraints**: save the current constraints in the [JSON format](../user_guide/constraints.md) to a specific path. Constraint JSON files can also be load into the viewer. + - **Example**: allows saving a new example that encompasses the current motion, constraints, and all settings. This is useful for reloading previous work. If examples are saved to the demo examples directory, they will be loadable from the Examples dropdown menu, otherwise you can load them through file path in this menu. + +- **Exports** + - **Screenshot**: save current canvas as an image that can be downloaded through your browser + - **Video**: record the current motion to a video that can be download through your browser + - **Motion**: save the current motion to a format of your choice depending on the loaded skeleton: + - SOMA: `NPZ` or `BVH` + - G1: `NPZ` or `CSV` + - SMPL-X: `NPZ` or `AMASS NPZ` + These formats are described in [output formats](../user_guide/output_formats.md). diff --git a/docs/source/interactive_demo/generation.md b/docs/source/interactive_demo/generation.md new file mode 100644 index 0000000000000000000000000000000000000000..dd9b5c7222c9d6c7b2b7c2fc67624e903f722f9c --- /dev/null +++ b/docs/source/interactive_demo/generation.md @@ -0,0 +1,13 @@ +# Generation + +The most important panel is the "Generate" which allows you to call Kimodo to generate one or more motions based on the prompts, constraints, and settings provided. + +![Generate panel](../_static/demo/generate_panel.png) + +- **Num Samples**: the number of motions to generate based on the current settings. When multiple samples are generated, you _must_ choose a single sample by clicking the character in the viewer before editing constraints or generating new motion. +- **SOMA Layer**: if using a `Kimodo-SOMA` model, this option will appear. It allows you to use the SOMA body layer to skin the character instead of using the SOMA rig. For details on the difference between the two, see the [Skeletons page](../key_concepts/skeleton.md#soma-default). +- **Seed**: random seed for repeatable generation +- **Denoising steps**: number of steps to use with DDIM +- **CFG Text/Constraint Weight**: the weights to use for classifier-free guidance +- **Post-Processing**: whether to use foot skate cleanup and constraint post-optimization to improve motion after generation + - **Root Margin**: if the skeleton root deviates more than this margin from a constraint, the post-processing will fix it diff --git a/docs/source/interactive_demo/index.md b/docs/source/interactive_demo/index.md new file mode 100644 index 0000000000000000000000000000000000000000..879fdaf7531a23439c6a934105f7c3da108e5bc8 --- /dev/null +++ b/docs/source/interactive_demo/index.md @@ -0,0 +1,48 @@ +# Interactive Demo + +The web-based interactive demo provides an intuitive interface for generating motions with any of the Kimodo model variations. + +![Demo Interface](../_static/overview.png) +*Interactive demo interface build with [Viser](https://github.com/viser-project/viser)* + +```{note} +To see the demo in action, follow the [setup instructions](launching.md) below and launch it locally. After launching, open the demo in a web browser at http://127.0.0.1:7860 or use port forwarding if running on a server. +``` + +The demo provides a timeline-based interface for composing text prompts and +constraints, with real-time 3D visualization. Here are some key features: + +- **Multiple Characters**: Supports generating with the SOMA, G1, and SMPL-X versions of Kimodo +- **Text Prompts**: Enter one or more natural language descriptions of desired motions on the timeline +- **Timeline Editor**: Add and edit keyframes and constrained intervals on multiple constraint tracks +- **Constraint Types**: + - Full-Body: Complete joint position constraints at specific frames + - 2D Root: Define waypoints or full paths to follow on the ground plane + - End-Effectors: Control hands and feet positions/rotations +- **Constraint Editing**: Editing mode allows for re-posing of constraints or adjusting waypoints +- **3D Visualization**: Real-time rendering of generated motions with skeleton and skinned mesh options +- **Playback Controls**: Preview generated motions with adjustable playback speed +- **Multiple Samples**: Generate and compare multiple motion variations +- **Examples**: Load pre-existing examples to better understand Kimodo's capabilities +- **Export**: Save constraints and generated motions for later use + + +## Quick Links + +- [Starting the Demo](launching.md) +- [UI Overview](ui_overview.md) +- [Examples](examples.md) + + +```{toctree} +:maxdepth: 2 +:hidden: + +launching +ui_overview +model_selection +examples +generation +constraints +export_results +``` diff --git a/docs/source/interactive_demo/launching.md b/docs/source/interactive_demo/launching.md new file mode 100644 index 0000000000000000000000000000000000000000..48e40cb6ab67e30fb8a3136d5542f9318a00f217 --- /dev/null +++ b/docs/source/interactive_demo/launching.md @@ -0,0 +1,84 @@ +# Running the Demo + +After following the installation [instructions](../getting_started/installation.md), the demo can be launched with the commands below. The demo runs in the web browser at [http://localhost:7860](http://localhost:7860). + + + +
+If you run the demo on a server, you can use port forwarding to access it. + +To access the demo's web interface when running on a remote server, set up SSH port forwarding so your web browser can reach `http://localhost:7860` as if it was local. + +**Option 1: Add LocalForward to your SSH config** + +Edit (or create) your SSH config file (typically `~/.ssh/config`): + +``` +Host your-server-name + HostName your.server.address + User username + LocalForward 7860 localhost:7860 +``` +Then connect with: +``` +ssh your-server-name +``` + +**Option 2: Use the SSH command-line directly** + +From your local machine, run: +``` +ssh -N -L 7860:localhost:7860 username@your.server.address +``` +This will forward your local port 7860 to the server's port 7860. +After connecting, open [`http://localhost:7860`](http://localhost:7860) in your web browser. + +Replace `username` and `your.server.address` with your actual user and server info. + +
+
+ +If you will be restarting the demo frequently, we recommend first starting the text encoder service in the background, as detailed in the [quick start guide](../getting_started/quick_start.md#run-text-encoder-service). If the text encoder service is not running, the demo will automatically load the text encoder model. + +The demo will also automatically download the Kimodo model checkpoint on launch and whenever needed when the model preference is changed in the UI. + +## Launch from Command Line +If you installed Kimodo as a package or from source, the demo can be started with: +```bash +kimodo_demo +``` + +## Launch with Docker +If you installed with Docker, you can start the demo with: +```bash +docker compose up demo +``` + +
+Additional Tips for Docker + +You may find the following commands useful if running Kimodo within the Docker containers. In the example commands below, you can also replace `demo` by `text-encoder`: + +**Check logs:** + +```bash +docker compose logs demo +``` + +**Stop service:** + +```bash +docker compose stop demo +``` + +**Restart service:** + +```bash +docker compose restart demo +``` + +**Stop and remove everything:** + +```bash +docker compose down +``` diff --git a/docs/source/interactive_demo/model_selection.md b/docs/source/interactive_demo/model_selection.md new file mode 100644 index 0000000000000000000000000000000000000000..2d765ebd98fa5680c74d1aab649162ec19352c1a --- /dev/null +++ b/docs/source/interactive_demo/model_selection.md @@ -0,0 +1,16 @@ +# Model Selection + +Model selection allows choosing between the Kimodo models detailed in the [quick start guide](../getting_started/quick_start.md#overview-kimodo-models). + +The models determine which character is loaded in the scene and the possible export options. + +- **SOMA**: default human skeleton +- **G1**: MuJoCo-compatible exports +- **SMPL-X**: SMPL-X compatible outputs + +For details on each skeleton, see [Skeletons](../key_concepts/skeleton.md). + +Model selection UI + + +![Skeleton overview](../_static/skeletons/skeletons.png) diff --git a/docs/source/interactive_demo/ui_overview.md b/docs/source/interactive_demo/ui_overview.md new file mode 100644 index 0000000000000000000000000000000000000000..ac6741aca921432f710bfb77149c157cf1072faf --- /dev/null +++ b/docs/source/interactive_demo/ui_overview.md @@ -0,0 +1,63 @@ +# UI Overview + +This page gives an overview of each of the main elements of the demo UI and how to use them. + +![Demo Interface](../_static/overview.png) +*An example scene within the demo webapp* + +## Viewer +![Viewer](../_static/demo/viewer.png) + +The 3D viewer shows the currently generated motion. It supports skeleton or skinned mesh rendering, which is configurable in the "Visualize" panel. + +### Camera +- **Left-drag**: rotate +- **Right-drag**: pan +- **Scroll**: zoom + +### Playback +- **Space** to play/pause +- **←/→** to step frames, or click the frame number. + +## Timeline + +![Timeline](../_static/demo/timeline.png) + +The timeline is where you: + +- add, edit, and delete **prompt segments** +- add and delete **constraints** at frames or intervals and adjust timing + +### Timeline Navigation +- **Scroll Up/Down** in the timeline: move left/right +- **Shift + Scroll** in the timeline: zoom in/out + +### Prompts +- **Double-Click** a text prompt to edit the text +- **Click and Drag** the right edge of a prompt box to extend/shorten it (2-10 sec) +- **Click Empty Space** to add a prompt +- **Right-Click** a prompt to delete it + +### Constraints +Constraints can be added after generating for the first time when there is an active motion in the viewer: +- **Click** in the timeline tracks (Full-Body / 2D root etc) to add a constraint of that type using the pose at that frame +- **Ctrl/Cmd + Click + Drag** to add an interval constraint, or expand a keyframe into an interval +- **Click + Drag** an existing constraint to move it to a different frame +- **Right-Click** on a constraint to delete it +- To **edit** a constraint: + - Move playback to the target frame + - Click **Enter Editing Mode** in the Constraints tab of the Settings Panel. Note you must exit editing mode before generating again. + + +## Settings Panel +![Panel](../_static/demo/panel.png) + +The settings panel includes: +- model selection +- loading examples +- model parameter selection for generation and post-processing +- parameters for constraint editing +- motion loading and saving +- visualization options + +Important settings panels are individually explained on subsequent pages. diff --git a/docs/source/key_concepts/constraints.md b/docs/source/key_concepts/constraints.md new file mode 100644 index 0000000000000000000000000000000000000000..f1930fb6ac19230a313346176b60b78bb16b7e3f --- /dev/null +++ b/docs/source/key_concepts/constraints.md @@ -0,0 +1,52 @@ +# Constraints + +Constraints are time-localized signals that steer the generated motion toward +specific spatial goals while keeping the rest of the motion free for the model +to resolve. You can combine constraints with text prompts to control trajectory, +pose, and end-effectors. Constraints are most easily defined in the [interactive demo](../interactive_demo/constraints.md) and can be saved to the [JSON format](../user_guide/constraints.md). + +![Overview diagram of constraint types on a timeline](../_static/constraints.png) + +## Why Constraints? + +Constraints allow you to: + +- pin the character to a target pose or keyframe +- guide a path on the ground while preserving natural motion +- fix hands or feet at specific times (for example, touch or contact events) + +## Constraint Types + +Kimodo is trained to excel at specific types of constraints. + +**Sparse root 2D waypoint**: ground-plane 2D waypoints that guide the global translation of the character. This constrains the 2D components of the smoothed root representation generated by Kimodo. + +**Dense root 2D path**: dense 2D path constraints that guide a continuous trajectory. This constrains the 2D components of the smoothed root representation generated by Kimodo. + +**Sparse full-body keyframe**: full-body pose targets at specific frames. Within the Kimodo motion representation, this constrains the smoothed root position and all body joint positions at a specific frame. + +**Sparse end-effector constraint**: hands or feet targets while leaving the rest of the body flexible. This constrains the smoothed root position along with the specified end-effectors. For hands, this will constrain the wrist position and rotation along with the hand end position. For feet, it constraints the heel position and rotation along with the toe position. Kimodo is trained to support arbitrary subsets of end-effectors. + +**Foot contacts**: toe/heel contact patterns. While the model is trained to support this, it is not currently implemented in the demo UI or Python API. + + +```{note} +For SOMA models, constraints may be authored or displayed on the full `somaskel77` skeleton, but Kimodo converts them to the reduced `somaskel30` representation before passing them to the model. See the [skeleton](./skeleton.md) section for more details. +``` + +## Coordinate Space + +All constraint values are in a **Y-up** coordinate system with units in **meters**. The model expects constraints relative to a canonical origin where the root starts at XZ = (0, 0) at frame 0. The initial heading can be set via the `first_heading_angle` generation parameter (defaults to 0, facing +Z). See the [constraints JSON format](../user_guide/constraints.md#coordinate-space-and-units) for full details on each field. + +## Time and Scope + +In our CLI and demo, constraints can be defined at: + +- **Single frames**: keyframe-style constraints +- **Intervals**: guidance across a range of frames + +However, as described above, the model is trained to excel mostly at sparse keyframes, with dense keyframes usually only seen for root paths. See [best practices](./limitations.md) for more details. + +## Post-Processing + +Since it is very challenging for a neural network to strictly adhere to constraints, the demo and CLI support motion post-processing to ensure motion _exactly_ hits constraints. This is done through a lightweight optimization that smoothly adjusts joints while minimizing changes in acceleration and velocity. diff --git a/docs/source/key_concepts/limitations.md b/docs/source/key_concepts/limitations.md new file mode 100644 index 0000000000000000000000000000000000000000..5632b7e7bbc4d72b3361d072164bd6a3815e1da9 --- /dev/null +++ b/docs/source/key_concepts/limitations.md @@ -0,0 +1,23 @@ +# Best Practices + +On this page, we summarize the best approaches to maximize Kimodo's capabilities in terms of prompting and constraints, and also summarize known limitations and failure cases. For additional context, please see the [tech report](https://research.nvidia.com/labs/sil/projects/kimodo/assets/kimodo_tech_report.pdf). + +## Text Prompting +- For best results, begin each prompt with "A person..." (e.g., "A person walks forward" or "A person jumps and waves"). This phrasing helps clarify the subject and intent of the motion, and is more closely aligned with the style of prompts used in the training data. The subject can also be stylized to better describe the motion such as "An old person..." or "A drunk person..." +- Keep each prompt focused one or at most two behaviors. For long sequences of action, split them into multiple prompts and generate in sequence. +- It's best to use a medium level of detail when describing a motion. Prompts like "A person walks." are too short and vague, while very long prompts describing detailed motion of each body part will be too much for the model to handle. Most training data is a middleground between these two. We recommend looking at the prompts in the [BONES-SEED dataset](https://huggingface.co/datasets/bones-studio/seed) to get an idea of prompt granularity. +- Kimodo is trained on a specific set of human behaviors. The training data tends to cover locomotion, gestures, everyday activities, common object interactions, videogame combat, dancing, and various styles including tired, angry, happy, sad, scared, drunk, injured, stealthy, old, and childlike. Prompts for actions outside of these categories will likely give bad results. For example "A baseball player walks up to the plate and swings a bat" is not good, becuase Kimodo has not trained on baseball data. +- When using multiple prompts (e.g., in the timeline UI), make sure each prompt has enough information on its own. For example, if prompt 1 is "A person is walking while carrying an object", then prompt 2 could be "A person walking carrying an object comes to a stop". If prompt 2 were instead "Then the person stops", the model will not have enough context for what happened previously and may generat poor quality motions. + +## Constraints +- Avoid using constraints that contradict the given text prompt or other types of constraints. If you are having trouble with a tradeoff between constraint and text accuracy, try adjusting the [classifier-free guidance weights](../user_guide/configuration.md). +- Except for dense 2d root paths, Kimodo is mainly trained to handle sparse temporal constraints. Kimodo will perform best when the number of constraints per constraint type is less than 20 keyframes. +- When foot contact accuracy and hitting constraints is high priority, make sure to enable [post-processing](./constraints.md#post-processing). + +## Limitations +- **Motion length:** Maximum generated motion duration is 10 sec per prompt +- **Number of constraints:** The number of constrained frames per constraint type should be less than 20 (excluding the root path constraint) +- **Overly long or complex prompts** can blur motion intent, especially when many distinct actions are packed into a single prompt. +- **Conflicting constraints:** can lead to artifacts or constraints that are ignored +- **Multi-prompt sequences**: When generating motions with a sequence of prompts, each motion is generated one at a time. The second motion is conditioned on the last frames of the first, so the transition between prompts actually happens at the start of the second motion. This means the second prompt must devote some of its duration to performing a smooth transition, which may reduce the time available to realize the new prompt content fully. +- **Post-processing**: The model by itself can generate foot skating and will not exactly hit constraints. Post-processing helps with this, but currently does not work well for the G1 robot skeleton. diff --git a/docs/source/key_concepts/model.md b/docs/source/key_concepts/model.md new file mode 100644 index 0000000000000000000000000000000000000000..979a97850e167e1e7cecfb3a685ff32b243f560a --- /dev/null +++ b/docs/source/key_concepts/model.md @@ -0,0 +1,24 @@ +# Model Overview + +At a glance: +- Input: text prompt + optional constraints. +- Output: full-body motion sequence +- Core Idea: denoise motion features with a two-stage transformer at each step. + +Kimodo is an explicit motion diffusion model that generates 3D human motion by denoising a sequence of skeleton poses. The model operates on a carefully designed motion representation that enables precise control over generated motion while minimizing common artifacts, such as floating and foot skating. The motion representation features a smoothed root that emulates paths drawn in practical animation tools, along with global joint rotations and positions amenable to sparse keyframe constraints. + +For full details, see the [tech report](https://research.nvidia.com/labs/sil/projects/kimodo/assets/kimodo_tech_report.pdf) + +![Kimodo model architecture](../_static/arch.png) + +## Diffusion Process + +At each step of the denoising process, the model takes in an embedding of the text prompt, a set of kinematic constraints, and the current noisy motion. Constraints are specified using the same motion representation as the input motion, and are used to overwrite the corresponding values in the noisy motion. Additionally, a mask indicating which elements are constrained is concatentated to the input motion. The goal is to predict a clean version of the input motion. + +## Two-Stage Transformer Denoiser + +Given these inputs, the two-stage transformer denoiser predicts a clean motion that aligns with the text and constraints. The two-stage denoiser decomposes root and body motion prediction: the root denoiser first predicts global root motion, which is transformed into a local representation as input to the body denoiser. The final output is the concatenation of the two stages. + +## Training Dataset + +A key component to effectively train Kimodo is the [Bones Rigplay](https://bones.studio/ai-datasets/) dataset, a large studio mocap dataset containing over 700 hours of production-quality human motion with corresponding text descriptions. The data covers locomotion, gestures, everyday activities, common object interactions, videogame combat, dancing, and various styles including tired, angry, happy, sad, scared, drunk, injured, stealthy, old, and childlike. diff --git a/docs/source/key_concepts/motion_representation.md b/docs/source/key_concepts/motion_representation.md new file mode 100644 index 0000000000000000000000000000000000000000..7070afc1a3711d250a19e2a6bde8fcd6fbf665f1 --- /dev/null +++ b/docs/source/key_concepts/motion_representation.md @@ -0,0 +1,40 @@ +# Motion Representation + +Kimodo uses a motion representation that combines a smoothed root representation with global joint positions, rotations, and various auxiliary features. +For full details, please refer to the [tech report](https://research.nvidia.com/labs/sil/projects/kimodo/assets/kimodo_tech_report.pdf). + +The representation is implemented in `kimodo/motion_rep/reps/kimodo_motionrep.py` and allows easily going to and from this feature representation. + +## Coordinate System + +All motion features use a right-handed coordinate system with: + +- **Y up** +- **+Z forward** + +## Smoothed Root Representation + +We use a smoothed root trajectory for the global root position to make +path-following constraints more natural and controllable. Smoothing removes +high-frequency pelvis jitter while preserving overall motion direction, so +2D waypoints or paths drawn by users remain clean and easy to match during +generation, while the pelvis can still move naturally around the smoothed +curve. + +![Comparison of smoothed root rep](../_static/smoothed_root.png) + +## Pose Feature + +At each frame, the pose feature vector is the concatenation of: + +- **Smooth root position** (`smooth_root_pos`, 3): Smoothed pelvis/root position. + The x/z components track ground-plane motion and y stores height. +- **Global root heading** (`global_root_heading`, 2): `[cos(theta), sin(theta)]` + heading direction of the root. +- **Local joint positions** (`local_joints_positions`, `J x 3`): Joint positions + in a pelvis-relative space with the smoothed root x/z offset applied. +- **Global joint rotations** (`global_rot_data`, `J x 6`): 6D rotation + representation of each joint's global orientation. +- **Joint velocities** (`velocities`, `J x 3`): Global joint velocities. +- **Foot contacts** (`foot_contacts`, 4): Binary contact indicators for the + left/right foot contact points. diff --git a/docs/source/key_concepts/skeleton.md b/docs/source/key_concepts/skeleton.md new file mode 100644 index 0000000000000000000000000000000000000000..1b1443a7a522239c75f3946685a1ee5c5e20f75e --- /dev/null +++ b/docs/source/key_concepts/skeleton.md @@ -0,0 +1,40 @@ +# Skeletons + +Different versions of Kimodo support different skeletons (character). A separate model is trained for each skeleton, with the +currently available options being [SOMA](https://github.com/NVlabs/SOMA-X), [G1](https://github.com/unitreerobotics/unitree_mujoco/tree/main/unitree_robots/g1), and [SMPL-X](https://github.com/vchoutas/smplx). + +The skeletons discussed on this page are defined in `kimodo/skeleton/definitions.py`. + +![Skeleton overview](../_static/skeletons/skeletons.png) + +## SOMA (default) + +SOMA is the default skeleton used for Kimodo. It it based on the [SOMA body model](https://github.com/NVlabs/SOMA-X), which is also used in the [BONES-SEED dataset](https://huggingface.co/datasets/bones-studio/seed). +Kimodo uses two closely related SOMA skeleton definitions: + +- **`somaskel30`**: the reduced 30-joint skeleton used internally by the model and by the core SOMA constraint formulation. It removes most finger and hand detail. +- **`somaskel77`**: the full 77-joint SOMA skeleton used for public-facing visualization and SOMA motion exports. + +In practice, Kimodo predicts SOMA motions on `somaskel30` and converts them to `somaskel77` when returning or visualizing results in the demo. Older assets and examples may still be stored on `somaskel30`, and the tooling keeps backward compatibility with those files. + +Note that all training data for Kimodo is on a uniform skeleton proportion corresponding to one single set of identity parameters for the SOMA body model. + +!["SOMA skeletons"](../_static/skeletons/soma_skels.png) + +Outputs on the SOMA skeleton can be visualized in two ways. The first is by articulating a fixed SOMA rig and doing traditional skinning (corresponds to `kimodo/viz/soma_skin.py` in the codebase). +Alternatively, we can take generated joint rotations and feed them through the SOMA layer with the set of identity parameters that correspond to the body shape of our uniform skeleton. An example of this in the codebase at `kimodo/viz/soma_layer_skin.py`, which uses the identity parameters defined from `kimodo/assets/skeletons/somaskel30/soma_base_fit_mhr_params.npz` (the same ones from BONES-SEED data). + +Due to peculiarities with data processing, using the SOMA rig and SOMA layer give very slightly different results in visualization, with the SOMA rig better reflecting the data that Kimodo was trained on. + +## Unitree G1 + +The G1 skeleton targets MuJoCo-compatible exports and robotics workflows. +The version that Kimodo uses is a 34-joint skeleton, with extra joints added for the toes to ease learning. When generated motions are exported to the MuJoCo `qpos` CSV format, these joints are removed to be compatible with downstream applications. + +G1 skeleton + +## SMPL-X + +This aligns with the SMPL-X model and supports AMASS-style exports. It uses 22 joints corresponding to only the body joints. This option is useful for compatibility with SMPL-X pipelines or downstream tools expecting AMASS parameters, but it is **not** the recommended Kimodo model to use since generated motions may display particularly severe retargeting artifacts. + +SMPL-X skeleton diff --git a/docs/source/project_info.md b/docs/source/project_info.md new file mode 100644 index 0000000000000000000000000000000000000000..aad8f839bc444d55b498deaaab04088483a4d6d2 --- /dev/null +++ b/docs/source/project_info.md @@ -0,0 +1,29 @@ +# Project Information + +## Citation + +If you use this code in your research, please cite: + +```bibtex +@article{Kimodo2026, + title={Kimodo: Scaling Controllable Human Motion Generation}, + author={Rempe, Davis and Petrovich, Mathis and Yuan, Ye and Zhang, Haotian and Peng, Xue Bin and Jiang, Yifeng and Wang, Tingwu and Iqbal, Umar and Minor, David and de Ruyter, Michael and Li, Jiefeng and Tessler, Chen and Lim, Edy and Jeong, Eugene and Wu, Sam and Hassani, Ehsan and Huang, Michael and Yu, Jin-Bey and Chung, Chaeyeon and Song, Lina and Dionne, Olivier and Kautz, Jan and Yuen, Simon and Fidler, Sanja}, + journal={arXiv:2603.15546}, + year={2026} +} +``` + +## License + +The codebase is licensed under Apache-2.0. Please see the codebase for full license text. Note that model checkpoints are licensed separately as indicated on the HuggingFace download pages. + +## Acknowledgments + +This project builds upon several excellent open-source projects: + +- [Viser](https://github.com/nerfstudio-project/viser) for 3D visualization +- [LLM2Vec](https://github.com/McGill-NLP/llm2vec) for text encoding + +## Contact + +For questions or issues, plese open an issue on this repository or reach out directly to the authors. diff --git a/docs/source/project_structure.md b/docs/source/project_structure.md new file mode 100644 index 0000000000000000000000000000000000000000..76ed5695de36a4e1bf6b62e632d56fe9b9a112a8 --- /dev/null +++ b/docs/source/project_structure.md @@ -0,0 +1,109 @@ +# Project Structure + +```text +kimodo/ +├── kimodo/ # Main Python package +│ ├── model/ # Model architecture and loading +│ │ ├── kimodo_model.py # Kimodo diffusion model wrapper +│ │ ├── twostage_denoiser.py # Two-stage denoising architecture +│ │ ├── backbone.py # Transformer encoder backbone +│ │ ├── diffusion.py # Diffusion process +│ │ ├── cfg.py # Classifier-free guidance +│ │ ├── common.py # Shared model utilities +│ │ ├── load_model.py # Model loading and registry lookup +│ │ ├── loading.py # Checkpoint loading utilities +│ │ ├── registry.py # Model registry (skeleton, checkpoint URLs) +│ │ ├── text_encoder_api.py # Text encoder API client +│ │ ├── tmr.py # TMR compatibility +│ │ └── llm2vec/ # LLM-based text encoder +│ ├── motion_rep/ # Motion representation +│ │ ├── reps/ # Skeleton-specific motion reps +│ │ │ ├── base.py # Base motion rep types +│ │ │ ├── kimodo_motionrep.py +│ │ │ └── tmr_motionrep.py +│ │ ├── conditioning.py # Conditioning (text, constraints) +│ │ ├── feature_utils.py # Feature extraction +│ │ ├── feet.py # Foot contact / smoothing +│ │ ├── smooth_root.py # Smooth root representation +│ │ └── stats.py # Normalization statistics +│ ├── skeleton/ # Skeleton definitions and kinematics +│ │ ├── definitions.py # Skeleton topology (joints, chains) +│ │ ├── registry.py # Skeleton registry +│ │ ├── base.py # Base skeleton types +│ │ ├── kinematics.py # Forward kinematics +│ │ ├── transforms.py # Rotation/transform utilities +│ │ └── bvh.py # BVH I/O +│ ├── viz/ # Visualization +│ │ ├── scene.py # 3D scene setup +│ │ ├── playback.py # Timeline / motion playback +│ │ ├── viser_utils.py # Viser 3D helpers +│ │ ├── gui.py # Demo GUI components +│ │ ├── constraint_ui.py # Constraint editing UI +│ │ ├── coords.py # Coordinate frames +│ │ ├── soma_skin.py # SOMA character skinning +│ │ ├── soma_layer_skin.py # SOMA layer-based skinning +│ │ ├── smplx_skin.py # SMPL-X skinning +│ │ └── g1_rig.py # G1 robot rig +│ ├── demo/ # Interactive web demo +│ │ ├── app.py # Demo entry (Gradio / Viser) +│ │ ├── config.py # Demo configuration +│ │ ├── state.py # Application state +│ │ ├── ui.py # UI layout and callbacks +│ │ ├── generation.py # Generation pipeline for demo +│ │ ├── embedding_cache.py # Cached text embeddings +│ │ ├── queue_manager.py # Request queue for demo +│ │ └── __main__.py # Demo run as module +│ ├── exports/ # Motion I/O and format conversion +│ │ ├── motion_io.py # Kimodo motion dict helpers (load, save, resample) +│ │ ├── motion_convert_lib.py # Library API for format conversion +│ │ ├── motion_formats.py # Format detection and FPS resolution +│ │ ├── bvh.py # SOMA BVH read/write +│ │ ├── mujoco.py # G1 MuJoCo qpos conversion +│ │ └── smplx.py # AMASS / SMPL-X conversion +│ ├── metrics/ # Evaluation metrics +│ │ ├── base.py # Metric base classes +│ │ ├── foot_skate.py # Foot skate metrics +│ │ ├── constraints.py # Constraint metrics +│ │ └── tmr.py # TMR-based metrics +│ ├── scripts/ # CLI and helper scripts +│ │ ├── generate.py # CLI for motion synthesis (kimodo_gen) +│ │ ├── motion_convert.py # CLI for format conversion (kimodo_convert) +│ │ ├── run_text_encoder_server.py # Text encoder server (kimodo_textencoder) +│ │ ├── gradio_theme.py # Gradio theme for demo +│ │ ├── lock_requirements.py # Dependency locking +│ │ └── mujoco_load.py # MuJoCo g1 csv loading +│ ├── assets/ # Package data (shipped with package) +│ │ ├── demo/ # Demo examples and config +│ │ └── skeletons/ # Skeleton assets +│ ├── constraints.py # Constraint definitions and handling +│ ├── geometry.py # Geometric utilities +│ ├── postprocess.py # Post-processing (e.g. MotionCorrection) +│ ├── meta.py # Motion metadata +│ ├── sanitize.py # Input sanitization +│ ├── assets.py # Asset path resolution +│ └── tools.py # General utilities +├── benchmark/ # Evaluation pipeline scripts +│ ├── create_benchmark.py # Step 1: Build test suite from SEED + metadata +│ ├── generate_eval.py # Step 2: Generate motions for test suite +│ ├── embed_folder.py # Step 3: Embed motions and text with TMR +│ ├── evaluate_folder.py # Step 4: Compute metrics for test cases +│ └── parse_folder.py # Step 5: Aggregate and display results +├── MotionCorrection/ # Optional C++/Python post-processing +│ ├── python/motion_correction/ # Python bindings +│ └── src/cpp/ # C++ implementation +├── docs/ # Documentation (Sphinx) +│ └── source/ # RST/MD sources +├── assets/ # Repo-level assets (banner, screenshots) +├── pyproject.toml # Package config and entry points +├── setup.py # Setuptools entry (if needed) +├── Dockerfile # Container image for demo +├── docker-compose.yaml # Docker Compose for demo + text encoder +└── README.md +``` + +Entry points (from `pyproject.toml`): + +- **`kimodo_gen`** — command-line motion synthesis (`kimodo.scripts.generate:main`) +- **`kimodo_demo`** — interactive web demo (`kimodo.demo:main`) +- **`kimodo_convert`** — motion format conversion (`kimodo.scripts.motion_convert:main`) +- **`kimodo_textencoder`** — text encoder server (`kimodo.scripts.run_text_encoder_server:main`) diff --git a/docs/source/user_guide/cli.md b/docs/source/user_guide/cli.md new file mode 100644 index 0000000000000000000000000000000000000000..4d8f200153d6c712e969c46117d8587e61a0fe93 --- /dev/null +++ b/docs/source/user_guide/cli.md @@ -0,0 +1,135 @@ +# Command-Line Interface + +The primary CLI entrypoint is the `kimodo_gen` command. This corresponds to the script located in `kimodo/scripts/generate.py`, therefore you can equivalently use `python -m kimodo.scripts.generate`. + +**Docker Usage**: If you set up Kimodo with Docker, you can instead run generation inside the Docker container, replacing `kimodo_gen XXX` with `docker compose run --rm demo kimodo_gen XXX`. If you will be running generation multiple times, it is better to start the `demo` container (e.g., in another terminal or in the background), and then run commands inside it with `docker compose exec demo kimodo_gen XXX`. + +**Single Prompt Generation:** + +```bash +kimodo_gen "A person walks forward." \ + --model Kimodo-SOMA-RP-v1 \ + --duration 5.0 \ + --output output +``` + +The `--model` command corresponds to the Kimodo model checkpoint to generate with. By default, the `Kimodo-SOMA-RP-v1` is used if not provided. The output motion will be saved using the stem name given by `--output` in the Kimodo [output format](../user_guide/output_formats.md). If generating with a G1 or SMPL-X model, you can also save to other output formats like MuJoCo qpos CSV file and AMASS NPZ format. + +For **offline conversion** between Kimodo NPZ, AMASS NPZ, SOMA BVH, and G1 MuJoCo CSV after generating, use `kimodo_convert` (see [Motion format conversion](motion_convert.md)). + +**Multi-Prompt Generation:** + +Generating from a sequence of text prompts can be achieved by using multiple sentences separated by periods with corresponding durations: + +```bash +kimodo_gen "A person walks forward. A person is walking backwards." \ + --duration "5.0 4.0" \ +``` + +This command will use Kimodo to generate each prompt in sequence, with constraints added to the beginning of the second prompt to ensure continuity with the first generated motion. You can control how many frames are used to blend consecutive motions with the `--num_transition_frames` argument. + +**Single Prompt with Constraints:** + +Generation can be constrained by providing a constraints JSON file (see the [Constraints Format Definition](constraints.md)). + +```bash +kimodo_gen "A person walks forward and picks something up from the ground." \ + --model Kimodo-SOMA-RP-v1 \ + --duration 5.0 \ + --constraints kimodo/assets/demo/examples/kimodo-soma-rp/03_full_body_keyframes/constraints.json +``` + +Constraint files can be created and saved from the interactive demo or manually defined following +the [constraints format guide](constraints.md). + +## Output Formats + +For full details on output formats, see [this page](output_formats.md). + +To convert between these formats offline, see [Motion format conversion](motion_convert.md) (`kimodo_convert`). + +CLI generation uses a single **output stem** (`--output`) for all formats (NPZ, AMASS NPZ, CSV, and BVH). It can write either **one file** or **a folder of files**, depending on the number of samples: + +- **One sample** (`--num_samples 1`): writes a single file per format at the stem (e.g. `--output test` → `test.npz`, `test.csv`). No folder is created. For SMPLX, AMASS is written to `test_amass.npz`. +- **Multiple samples**: creates a folder with that stem and writes one file per sample with suffixes `_00`, `_01`, etc. (e.g. `--output test` → `test/test_00.npz`, ...). + +Use the `--bvh` flag to also export BVH (SOMA only) to the same stem. + +### Output Rest Pose + +For SOMA-based Kimodo models, motions can be exported with respect to two different rest poses. The default rest pose, that is always used by the `NPZ` format, is a standard T-pose consistent with the canonical T-pose of the SOMA model. For `BVH` outputs, the default rest pose is a non-standard pose, but it is consistent with the BVH format of the [BONES-SEED dataset](https://huggingface.co/datasets/bones-studio/seed). To output a `BVH` file with the standard T-pose as the rest pose, you can use the `--bvh_standard_tpose` option. + +The standard T-pose used by Kimodo is available as a BVH file in the [repo assets](https://github.com/nv-tlabs/kimodo/tree/main/kimodo/assets/skeletons/somaskel77). + +## Visualizing Generated Motions + +Motions generated with the CLI can be visualized in the demo UI. To do this, under "Load/Save" > "Motion", type in the path of the generated output npz file, then click "Load Motion" to load it into the viewer. If you used constraints when generating, those can also be loaded in in a similar way. + +## Arguments + +To see all available flags, run `kimodo_gen --help`. They are: + +- `prompt`: Text description of the desired motion (required) +- `--model`: Model name to use (default: `Kimodo-SOMA-RP-v1`; options are the models in [this table](../getting_started/quick_start.md#overview-kimodo-models)) +- `--duration`: Motion duration in seconds (default: `5.0`). For multiple prompts, + pass space-separated durations in a string. +- `--diffusion_steps`: Number of denoising steps (default: `100`) +- `--num_samples`: Number of motion variations to generate (default: `1`) +- `--num_transition_frames`: Frames used to blend between prompts (default: `5`) +- `--constraints`: Path to a JSON file containing constraints +- `--output`: Output stem name (default: `output`). Used for all formats (NPZ, AMASS NPZ, CSV, BVH). With one sample, writes a single file per format (e.g. `test.npz`, `test.csv`). With multiple samples, creates a folder and writes `test_00.npz`, `test_01.npz`, … inside it. For SMPLX with one sample, AMASS is written to `stem_amass.npz` so it does not overwrite the main NPZ. +- `--save_example_dir`: If given, saves outputs to an "example" directory structure that can be loaded in the Kimodo demo. +- `--bvh`: Optional flag. When set, also export BVH (SOMA models only) using the same stem as `--output`. +- `--bvh_standard_tpose`: If exporting BVH, export with the rest pose being the standard T-pose rather than the rest pose consistent with the BONES-SEED dataset. +- `--seed`: Seed for reproducible results +- `--no-postprocess`: Disable post-processing (includes foot skate cleanup and constraint optimization) +- `--input_folder`: Folder containing meta.json and optional constraints.json. If set, generation settings are loaded from meta.json. These are found in demo example folders. +- `--cfg_type`: Classifier-free guidance mode: `nocfg`, `regular`, or `separated` (the custom mode with independent text and constraint scales). See {ref}`Classifier-free guidance (details) ` below. +- `--cfg_weight`: One float for `regular` CFG, or two floats `[text_weight, constraint_weight]` for `separated` CFG. If you pass only weights (no `--cfg_type`), one value implies `regular` and two imply `separated`. Not used with `nocfg`. + +:::{dropdown} Classifier-free guidance (CFG) +:name: classifier-free-guidance-cfg + +The CLI mirrors the Python API in [Generation parameters](configuration.md): Kimodo supports standard CFG (`regular`) and a **separated** variant with two scales—text vs. constraints—which is the usual setting in this project. + +**Rules:** + +- `nocfg`: no weights; do not pass `--cfg_weight`. +- `regular`: pass exactly one value after `--cfg_weight`. +- `separated`: pass exactly two values after `--cfg_weight`. + +If you pass **`--cfg_type` or `--cfg_weight` on the command line**, those values override any `cfg` block in `meta.json` when using `--input_folder`. If you omit both flags, `meta.json` may still supply CFG via `cfg.enabled`, `cfg.text_weight`, and `cfg.constraint_weight` (same shape as the interactive demo examples). If there is no CLI CFG and no `cfg` in meta, the model uses its built-in defaults. + +Examples: + +```bash +# No classifier-free guidance +kimodo_gen "A person walks." --cfg_type nocfg + +# Standard CFG (single scale) +kimodo_gen "A person walks." --cfg_type regular --cfg_weight 2.5 + +# Separated CFG (text scale, then constraint scale) +kimodo_gen "A person walks." --cfg_type separated --cfg_weight 2.0 1.5 + +# Infer mode from arity: one float -> regular; two floats -> separated +kimodo_gen "A person walks." --cfg_weight 2.0 2.0 +``` + +::: + +## Python API +The `kimodo/scripts/generate.py` script is a good place to start to familiarize yourself with the Python API of Kimodo if you'd like to use this directly. The full model API is detailed in the [API documentation](../api_reference/index.rst). + +If you want to use kimodo in another project, you can interact with it like this: + +```python +from kimodo import load_model + +model = load_model("kimodo-soma-rp", device="cuda") +output = model( + prompt="A person jumps", + num_frames=150, + num_denoising_steps=100, +) +``` diff --git a/docs/source/user_guide/configuration.md b/docs/source/user_guide/configuration.md new file mode 100644 index 0000000000000000000000000000000000000000..edb5d894407783b2a4fa3a327f6fa17767832c59 --- /dev/null +++ b/docs/source/user_guide/configuration.md @@ -0,0 +1,35 @@ +# Generation Parameters + +In the demo UI, command-line tool (`kimodo_gen` / `python -m kimodo.scripts.generate`), and low-level Python API, Kimodo allows some advanced configuration for motion generation. + +## Classifier-Free Guidance + +Control the strength of text and constraint guidance: + +```python +output = model( + prompt="A person jumps", + num_frames=150, + cfg_weight=[2.0, 2.0], # [text_weight, constraint_weight] + cfg_type="separated", # Options: "nocfg", "regular", "separated" + num_denoising_steps=100, +) +``` + +These are helpful when there is a tradeoff between following the prompt and hitting constraints. + +The CFG options are: +- `cfg_type="nocfg"`: No guidance (faster, less controllable) +- `cfg_type="regular"`: "Standard" classifier-free guidance + - Equation: `out_uncond + w * (out_text_and_constraint - out_uncond)` +- `cfg_type="separated"`: Separate weights for text and constraints + - Equation: `out_uncond + w_text * (out_text - out_uncond) + w_constraint * (out_constraint - out_uncond)` + +### CLI + +The same options are available from the command line as `--cfg_type` and `--cfg_weight`. See the {ref}`CLI user guide (CFG) ` for examples, validation rules, and how `meta.json` interacts with explicit flags when using `--input_folder`. + +## Denoising Steps +The number of denoising steps used in DDIM sampling can be used to control the speed vs. quality trade-off: +- Fewer steps (50-100): Faster inference, slightly lower quality +- More steps (100-200): Higher quality, slower inference diff --git a/docs/source/user_guide/constraints.md b/docs/source/user_guide/constraints.md new file mode 100644 index 0000000000000000000000000000000000000000..ce3040fc0fb6b85fa1685c6ca3600960f4850462 --- /dev/null +++ b/docs/source/user_guide/constraints.md @@ -0,0 +1,111 @@ +# Constraints JSON Format + +The `--constraints` flag in the CLI expects a JSON file containing a list of constraint objects. +It is easiest to look at the examples provided with the demo to see how these are formatted. These can be seen for various model types in `kimodo/assets/demo/examples`. + +> Tip: the easiest way to get a valid constraints file is to create constraints in the interactive demo and to click on `Save Constraints`. + +## High-Level Structure + +- The file is a JSON array: `[{...}, {...}, ...]` +- Each element is an object with at least: + - `type` (string) + - `root2d`, `fullbody`, `left-hand`, `right-hand`, `left-foot`, `right-foot`, `end-effector` + - `frame_indices` (array of integers): 0-based frame indices within the generated clip. + + +```{note} +For SOMA models, constraints may be authored or displayed on the full `somaskel77` skeleton, but Kimodo converts them to the reduced `somaskel30` representation before passing them to the model. See the [skeleton](../key_concepts/skeleton.md) section for more details. +``` + +## Coordinate Space and Units + +All spatial values in constraints use the same coordinate system as Kimodo's internal motion representation: + +- **Axes**: **Y-up**, with locomotion on the **XZ ground plane**. The Y axis points up, X and Z span the horizontal ground plane. +- **Units**: **Meters**. Joint positions, root translations, and 2D root coordinates are all in meters. + +### Canonicalization + +During training, every motion is *canonicalized* so that the (smoothed) root starts at the XZ origin `(0, 0)` at frame 0. +The initial body heading (facing direction) is randomly rotated and passed to the model as an explicit input (`first_heading_angle`), so the model is robust to arbitrary initial orientations. + +At inference, constraints should be authored **relative to this canonical origin**: +- `smooth_root_2d` values at frame 0 should be at `(0, 0)`, with subsequent frames expressing displacement from there. +- `root_positions` XZ components follow the same convention; Y is the **absolute hip height above the ground** (typically ~0.9 m for a standing pose, lower for crouching/sitting). +- `first_heading_angle` (a generation parameter, not part of the constraints JSON) defaults to `0.0` radians (facing +Z) but can be set to any value to change the initial facing direction. + +### Field-specific notes + +| Field | Space | Notes | +|-------|-------|-------| +| `smooth_root_2d` | `[x, z]` ground plane (meters) | Relative to the canonical origin. | +| `root_positions` | `[x, y, z]` (meters) | Y is absolute hip height above ground. XZ relative to canonical origin. | +| `global_root_heading` | `[cos(θ), sin(θ)]` | **Not** a raw radian value — must be a 2-element cosine/sine pair per frame (i.e. the heading direction vector). | +| `local_joints_rot` | axis-angle (radians) | Local joint rotations in the skeleton's rest-pose frame. | + +### Constraints not at frame 0 + +Adding a constraint at frame 0 is **not** required. If the first constrained frame is later in the sequence (e.g. frame 45), Kimodo generates the initial frames freely from its learned distribution, starting near XZ = (0, 0) with the heading set by `first_heading_angle`. The constraint just needs to be reachable from that starting configuration given the text prompt and motion duration. + +## Constraint Types +Depending on `type`, additional fields are required or optional. All numeric arrays are plain nested JSON lists. In the following definitions `T` is the number of constrainted frames (i.e., number of `frame_indices`) and `J` is the number of skeleton joints. + + +### `root2d` +This captures 2D root waypoints and 2D root paths. It requires: + +- `smooth_root_2d` (array shapes `[T, 2]`): Smoothed root positions `[x, z]` on the ground plane at the given `frame_indices`. + +and optionally: +- `global_root_heading` (array shapes `[T, 2]`): Global root heading direction `[cos, sin]` at the given `frame_indices`. + +### `fullbody` +This captures full-body keyframe constraints on joint positions. It includes: + +- `local_joints_rot` (array shaped `[T, J, 3]`): Per-frame per-joint **axis-angle** local rotations (radians). Constraint joint positions will be derived from these. +- `root_positions` (array shaped `[T, 3]`): Root (hips) translation `[x, y, z]`. +- `smooth_root_2d` (optional; array of `[T, 2]`): Smoothed root positions `[x, z]`. If omitted, it is taken as the `[x, z]` components of `root_positions`. + +Note the `local_joint_rot` will not explicitly be constrained, the constraint will be on the joint positions that results from FK with the given joint rotations. + +### `left-hand` / `right-hand` / `left-foot` / `right-foot` +Captures end-effector constraints on the hand/feet joint positions and global rotations. + +These use the same fields as `fullbody`. However, under the hood these will only affect the corresponding end-effectors and hips. Each of these types is a shorthand for `end-effector` with pre-set joint names. + +### `end-effector` +A general end-effector constraint that requires an additional field: + +- `joint_names` (array of strings): Which end-effectors to constrain (e.g. `["left_hand"]`, `["right_foot", "left_foot"]`). Available names depend on the skeleton; see the skeleton's `expand_joint_names()` for the full mapping. + +Otherwise uses the same fields as `fullbody` (`local_joints_rot`, `root_positions`, optional `smooth_root_2d`). + +## Examples + +### Root 2D waypoints + +```json +[ + { + "type": "root2d", + "frame_indices": [0, 30, 60], + "smooth_root_2d": [[0.0, 0.0], [0.5, 0.0], [1.0, 0.1]] + } +] +``` + +### Full-body keyframe + +```json +[ + { + "type": "fullbody", + "frame_indices": [60], + "root_positions": [[0.0, 0.96, 1.5]], + "local_joints_rot": [[[0.0, 0.0, 0.0], "... one [3] per joint ..."]] + } +] +``` + +Here `root_positions` places the hips at x=0, y=0.96 m (standing height), z=1.5 m forward from the origin. `local_joints_rot` is a `[T, J, 3]` array of axis-angle rotations for every joint in the skeleton. diff --git a/docs/source/user_guide/motion_convert.md b/docs/source/user_guide/motion_convert.md new file mode 100644 index 0000000000000000000000000000000000000000..1358e233311258c3fce1a68a8a54d35900e3191e --- /dev/null +++ b/docs/source/user_guide/motion_convert.md @@ -0,0 +1,76 @@ +# Motion Format Conversion + +The `kimodo_convert` command converts between the formats described in [Output formats](output_formats.md): **Kimodo NPZ**, **AMASS NPZ** (SMPL-X), **SOMA BVH**, and **G1 MuJoCo CSV**. + +## Frame rate (30 Hz Kimodo NPZ) + +Any conversion **to Kimodo NPZ** (from AMASS, SOMA BVH, or G1 CSV) **writes motion at 30 Hz**, matching Kimodo’s common generation rate. If the detected source rate differs, the tool **resamples** along time, then derived channels (contacts, smooth root, heading) are recomputed via forward kinematics. + +If resampling is required, a **warning** is emitted with the assumed source rate, input/output frame counts, and a reminder that `--source-fps` sets the **source** rate if autodetection is wrong. When the source is already ~30 Hz with the same frame count, no warning is shown (motion is only re-derived via FK for consistency). + +
+Resampling strategy details + +The resampler picks one of two strategies based on the ratio `source_fps / target_fps`: + +- **Integer-ratio fast path** — When the ratio is close to an integer ≥ 2 (within a tolerance of 0.05), the resampler simply takes every *step*-th frame (`frames[::step]`). For example, 120 Hz → 30 Hz has ratio 4, so every 4th frame is kept. This is exact and very fast. +- **Interpolation fallback** — Otherwise, the output timeline is linearly spaced over the input range. Root positions are linearly interpolated, and local joint rotations are interpolated via quaternion slerp. This handles arbitrary rate conversions (e.g. 50 Hz → 30 Hz). + +In both cases, `complete_motion_dict` is re-run at the target rate so that all derived channels (velocities, foot contacts, heading, smooth root) stay consistent with the new frame spacing. + +
+ +## Usage + +```bash +kimodo_convert INPUT OUTPUT [options] +``` + +Formats are inferred from file extensions and (for `.npz`) from file contents. You can override with `--from` and `--to`. + +### Supported conversions + +| From | To | Notes | +|------|-----|--------| +| AMASS `.npz` | Kimodo `.npz` | SMPL-X, 22 joints. Uses `--z-up` by default (same as Kimodo’s AMASS export). | +| Kimodo `.npz` | AMASS `.npz` | Requires `local_rot_mats` with 22 joints (SMPL-X). | +| SOMA `.bvh` | Kimodo `.npz` | Expects a **Kimodo-exported** SOMA BVH (same hierarchy as `save_motion_bvh`). If the BVH uses the standard T-pose as rest pose, pass in `--bvh_standard_tpose`. | +| Kimodo `.npz` | SOMA `.bvh` | Accepts 77 joints (SOMA full) or 30 joints (somaskel30, auto-expanded to 77 with relaxed-hand rest poses). If you want the output BVH to use the standard T-pose as rest pose, pass in `--bvh_standard_tpose`. | +| G1 `.csv` | Kimodo `.npz` | Rows of shape `(36,)` = root xyz + root quat + 29 joint angles (see [output_formats](output_formats.md#csv-format-for-kimodo-g1)). | +| Kimodo `.npz` | G1 `.csv` | Requires 34 joints (G1). | + +### Common options + +- **`--source-fps`**: Source motion frame rate in Hz (used before resampling to 30 Hz for Kimodo NPZ). If omitted, the tool auto-detects from `mocap_frame_rate` (AMASS), `Frame Time` (BVH), or defaults to **30** Hz. The legacy `--fps` alias is still accepted for backward compatibility. +- **`--no-z-up`**: For AMASS, disable the Y-up ↔ Z-up transform (treat data as already in Kimodo Y-up, +Z forward). +- **`--mujoco-rest-zero`**: For G1 CSV, match the `mujoco_rest_zero` flag used when the CSV was written (see `MujocoQposConverter.dict_to_qpos`). +- **`--bvh_standard_tpose`**: If input or output is BVH: the BVH file uses the standard T-pose as its rest pose instead of the BONES-SEED rest pose. + +### Examples + +```bash +# AMASS → Kimodo NPZ +kimodo_convert motion_amass.npz motion_kimodo.npz + +# Kimodo NPZ → AMASS +kimodo_convert motion_kimodo.npz motion_out_amass.npz + +# Kimodo SOMA NPZ → BVH +kimodo_convert motion_kimodo.npz motion.bvh + +# BVH → Kimodo NPZ +kimodo_convert motion.bvh motion_kimodo.npz + +# G1 CSV → Kimodo NPZ +kimodo_convert motion.csv motion_kimodo.npz + +# Kimodo G1 NPZ → CSV +kimodo_convert motion_kimodo.npz motion.csv +``` + +When both input and output are `.npz`, the tool assumes **AMASS → Kimodo** if the input is AMASS, and **Kimodo → AMASS** if the input is already a Kimodo NPZ. Use `--from` / `--to` if you need to disambiguate. + +## Limitations + +- **BVH import** is intended for BVHs produced by Kimodo (`Root` wrapper + SOMA77 joint names) and is also compatible with the BONES-SEED dataset, which uses the same skeleton hierarchy. Arbitrary BVH files with different joint names or hierarchies may not work. +- **G1 CSV** encodes only the degrees of freedom exposed in MuJoCo; the inverse path reconstructs local rotations from those angles (same convention as `to_qpos`). diff --git a/docs/source/user_guide/output_formats.md b/docs/source/user_guide/output_formats.md new file mode 100644 index 0000000000000000000000000000000000000000..39c11ad86333c412cdef8c486b48a78ef10941a1 --- /dev/null +++ b/docs/source/user_guide/output_formats.md @@ -0,0 +1,73 @@ +# Output Formats + +## Converting Between Formats + +To convert between the formats described below, see [Motion format conversion](motion_convert.md) (`kimodo_convert`). + +## Kimodo NPZ Format + +Generated motions are stored as NPZ files (one file per sample, e.g. `motion_00.npz`) containing: + +- `posed_joints`: Global joint positions `[T, J, 3]` +- `global_rot_mats`: Global joint rotation matrices `[T, J, 3, 3]` +- `local_rot_mats`: Local (parent-relative) joint rotation matrices `[T, J, 3, 3]` +- `foot_contacts`: Foot contact labels [left heel, left toe, right heel, right toes] `[T, 4]` +- `smooth_root_pos`: Smoothed root representations outputted from the model `[T, 3]` +- `root_positions`: The (non-smoothed) trajectory of the actual root joint (e.g., pelvis) `[T, 3]` +- `global_root_heading`: The heading direction output from the model `[T, 2]` + +Where: + +- `T`: number of frames +- `J`: number of joints in the exported skeleton representation (`77` for SOMA NPZ exports, `34` for G1, `22` for SMPL-X) + +If multiple samples are generated, files are saved with suffixes like `_00`, `_01`, etc. + +For SOMA models, the exported NPZ uses the full **`somaskel77`** skeleton even though the model itself operates internally on the reduced **`somaskel30`** skeleton. This means the saved `posed_joints`, `global_rot_mats`, and `local_rot_mats` arrays are written in the 77-joint SOMA layout. Older 30-joint SOMA NPZ files may still exist and remain loadable for backward compatibility. + +Also for SOMA models, the output motion is saved such that the rest pose (i.e. zero pose) is the standard T-pose that Kimodo uses internally. This differs from the default behavior of BVH export (see below), which uses a rest pose consistent with the BONES-SEED dataset format. The standard T-pose as a BVH file is also available [in the assets of the repo](https://github.com/nv-tlabs/kimodo/tree/main/kimodo/assets/skeletons/somaskel77). + +## BVH Format for Kimodo-SOMA + +When using a SOMA model and passing the `--bvh` flag to CLI generation, Kimodo also writes a BVH file alongside the NPZ output. + +- BVH export is supported for **SOMA models only** +- the exported hierarchy uses the full **`somaskel77`** skeleton +- if the motion is still in internal `somaskel30` form, Kimodo converts it to `somaskel77` before writing the BVH +- the file stores root translation plus per-joint local rotations for the clip at the generated frame rate +- by default, the rest pose (i.e., zero pose) of the saved BVH file is consistent with the BONES-SEED dataset format. If you prefer a standard T-pose as the rest pose, pass in `--bvh_standard_tpose` when generating. + +The exporter writes a standard plain-text BVH file and scales joint offsets and root motion from meters to centimeters (same format as the SEED dataset release). If multiple samples are generated, files are saved with suffixes like `_00`, `_01`, etc. + +## CSV Format for Kimodo-G1 + +When using `Kimodo-G1` models and providing `--output` to CLI generation, the exporter writes MuJoCo `qpos` +data to a CSV file. Each row corresponds to a pose in the motion and contains 36 values: + +- Root translation `[x, y, z]` +- Root rotation quaternion `[w, x, y, z]` +- 29 joint 1-DoF values (in G1 joint order) + +The CSV uses the MuJoCo coordinate system (z-up, +x forward). If multiple samples are generated, files are saved with suffixes like `_00`, `_01`, etc. + + +## AMASS NPZ Format for Kimodo-SMPLX + +When using the `Kimodo-SMPLX-RP` model and `--output` is specified to CLI generation, the exporter writes an +AMASS-style SMPL-X `.npz` file. Keys include: + +- `trans`: Root translation `[T, 3]` +- `root_orient`: Root orientation axis-angle `[T, 3]` +- `pose_body`: Body pose axis-angle `[T, 63]` (21 joints x 3) +- `pose_hand`: Hand pose axis-angle `[T, 90]` (15 joints x 2 hands x 3) +- `pose_jaw`: Jaw pose axis-angle `[T, 3]` +- `pose_eye`: Eye pose axis-angle `[T, 6]` +- `betas`: Shape coefficients +- `num_betas`: Number of shape coefficients +- `gender`: `neutral` +- `surface_model_type`: `smplx` +- `mocap_frame_rate`: Frame rate (fps) +- `mocap_time_length`: Motion duration in seconds + +The exporter converts from the Kimodo coordinate system (y-up, +z forward) +to AMASS coordinates (z-up, +y forward). If multiple samples are generated, files are saved with suffixes like `_00`, `_01`, etc. diff --git a/docs/source/user_guide/seed_dataset.md b/docs/source/user_guide/seed_dataset.md new file mode 100644 index 0000000000000000000000000000000000000000..58de9cb03e45532fb5adb8ab395f35b19ed94355 --- /dev/null +++ b/docs/source/user_guide/seed_dataset.md @@ -0,0 +1,90 @@ +# Loading BONES-SEED BVH data + +The [BONES-SEED dataset](https://huggingface.co/datasets/bones-studio/seed) is a publicly available optical motion-capture dataset distributed as BVH files with the [SOMA 77-joint skeleton](../key_concepts/skeleton.md). This page walks through the steps to parse a SEED BVH file and convert it into Kimodo's internal motion representation. + +This is a similar pipeline used by the benchmark to extract ground-truth motions from SEED data (see the [benchmark pipeline](../benchmark/pipeline.md)). + +## Step-by-Step Conversion + +### 1. Parse the BVH file + +`parse_bvh_motion` reads a BVH file and returns local joint rotation matrices, root translation (in meters), and the source frame rate. + +```python +from kimodo.skeleton.bvh import parse_bvh_motion + +local_rot_mats, root_trans, bvh_fps = parse_bvh_motion(bvh_path) +``` + +### 2. Subsample to 30 FPS + +Kimodo operates at 30 Hz. If the source BVH has a different frame rate (120 FPS for BONES-SEED), subsample by striding: + +```python +fps = 30 +step = round(bvh_fps / fps) +root_trans = root_trans[::step] +local_rot_mats = local_rot_mats[::step] +``` + +### 3. Convert to the standard T-pose + +The SEED BVH rest pose differs from Kimodo's canonical T-pose. The `to_standard_tpose` function remaps the local rotations accordingly and returns both local and global rotation matrices: + +```python +from kimodo.skeleton import SOMASkeleton77 + +skeleton = SOMASkeleton77() +local_rot_mats, global_rot_mats = skeleton.to_standard_tpose(local_rot_mats) +``` + +### 4. Compute Kimodo motion features + +Build the motion feature tensor used by the model. The feature layout is described in [Motion representation](../key_concepts/motion_representation.md). + +```python +from kimodo.motion_rep import KimodoMotionRep + +motion_rep = KimodoMotionRep(skeleton, fps) +feats = motion_rep(local_rot_mats, root_trans, to_normalize=False) +``` + +### 5. Canonicalize (optionally) and recover the motion dictionary + +Canonicalize so that the motion starts at the origin facing +Z, then invert the features back into a full motion dictionary: + +```python +can_feats = motion_rep.canonicalize(feats) +motion_dict = motion_rep.inverse(can_feats, is_normalized=False) +``` + +`motion_dict` is a dictionary with keys such as `local_rot_mats`, `global_rot_mats`, `posed_joints`, `root_positions`, `smooth_root_pos`, `foot_contacts`, etc. See [Output formats](output_formats.md) for details on the Kimodo NPZ layout. + +## Full script + +```python +from kimodo.motion_rep import KimodoMotionRep +from kimodo.skeleton import SOMASkeleton77 +from kimodo.skeleton.bvh import parse_bvh_motion + +# 1. Parse BVH +local_rot_mats, root_trans, bvh_fps = parse_bvh_motion(bvh_path) + +# 2. Subsample to 30 fps +fps = 30 +step = round(bvh_fps / fps) +root_trans = root_trans[::step] +local_rot_mats = local_rot_mats[::step] + +# 3. Convert to standard T-pose +skeleton = SOMASkeleton77() +local_rot_mats, global_rot_mats = skeleton.to_standard_tpose(local_rot_mats) + +# 4. Compute motion features +motion_rep = KimodoMotionRep(skeleton, fps) +feats = motion_rep(local_rot_mats, root_trans, to_normalize=False) + +# 5. Canonicalize and get the full motion dictionary +can_feats = motion_rep.canonicalize(feats) +motion_dict = motion_rep.inverse(can_feats, is_normalized=False) +``` diff --git a/kimodo/__init__.py b/kimodo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ed1fbe045195bcbb460c57abed1c87665eb50974 --- /dev/null +++ b/kimodo/__init__.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Kimodo: text-driven and constrained motion generation model.""" + +from .model.load_model import AVAILABLE_MODELS, DEFAULT_MODEL, load_model + +__all__ = [ + "AVAILABLE_MODELS", + "DEFAULT_MODEL", + "load_model", +] diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/01_single_text_prompt/meta.json b/kimodo/assets/demo/examples/kimodo-g1-rp/01_single_text_prompt/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..b55876632e72217a2b19d7d6cc7b47aa9d170aea --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/01_single_text_prompt/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person walking forward quickly stumbles but maintains their balance", + "duration": 5.0, + "num_samples": 1, + "seed": 43, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/01_single_text_prompt/motion.npz b/kimodo/assets/demo/examples/kimodo-g1-rp/01_single_text_prompt/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..d95ebb7873ab6aa80c00d417f32c9a3cff69c225 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/01_single_text_prompt/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e2f558a95c538b9a0230a156a0644f9a5d063fb5314c74d269adaecea41c566 +size 246198 diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/02_multi_text_ee_constraint/constraints.json b/kimodo/assets/demo/examples/kimodo-g1-rp/02_multi_text_ee_constraint/constraints.json new file mode 100644 index 0000000000000000000000000000000000000000..38dcfcc17edd4fb7c472f438be64491a966d38ea --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/02_multi_text_ee_constraint/constraints.json @@ -0,0 +1,752 @@ +[ + { + "type": "left-hand", + "frame_indices": [ + 40, + 155 + ], + "local_joints_rot": [ + [ + [ + -0.20672118663787842, + 0.0004979433142580092, + 0.010066316463053226 + ], + [ + 0.0789145976305008, + 0.0008333905134350061, + -5.267082087812014e-05 + ], + [ + -0.1686924546957016, + -0.0027884345036000013, + 0.0520743690431118 + ], + [ + 0.000989485066384077, + 0.1385614573955536, + 0.0005803265958093107 + ], + [ + 1.0274103879928589, + -0.0004089517460670322, + 0.0007986496202647686 + ], + [ + -0.39034226536750793, + -0.001306047779507935, + -4.922552761854604e-05 + ], + [ + 0.0023066187277436256, + -0.0007853881106711924, + -0.0062883589416742325 + ], + [ + 4.49517356173601e-05, + 0.0033443598076701164, + -0.0014551420463249087 + ], + [ + 0.07268467545509338, + -0.0011258760932832956, + -3.953919076593593e-05 + ], + [ + -0.1719113141298294, + 0.018712127581238747, + 0.06082615628838539 + ], + [ + 0.0011432868195697665, + 0.02744375728070736, + 0.0025501118507236242 + ], + [ + 0.41685307025909424, + -0.002692570211365819, + -0.0006283970433287323 + ], + [ + -0.1283608227968216, + 0.0030534265097230673, + 0.00016949126438703388 + ], + [ + -0.005590266548097134, + 0.0014076301595196128, + -0.038615260273218155 + ], + [ + -0.00013014793512411416, + 0.001360177993774414, + 6.41088408883661e-05 + ], + [ + 0.00010043015936389565, + -0.01370090153068304, + -0.00014910128084011376 + ], + [ + 0.00023336269077844918, + 0.0025421029422432184, + 0.04833226650953293 + ], + [ + 0.056574925780296326, + 0.0006874562823213637, + 0.0004548647266346961 + ], + [ + -0.37481847405433655, + -0.054357241839170456, + 0.2803272306919098 + ], + [ + 0.0013725318713113666, + 0.009074348025023937, + -0.0021504403557628393 + ], + [ + -0.0012184121878817677, + -0.4267229437828064, + 0.011203057132661343 + ], + [ + 1.255251407623291, + 0.0009449978824704885, + 0.0010158077348023653 + ], + [ + -0.003570390399545431, + -0.003947308287024498, + -0.5030224323272705 + ], + [ + 0.1881941556930542, + -0.000495748536195606, + 0.0016725400928407907 + ], + [ + -0.002223622752353549, + 0.11821465194225311, + 0.007546884939074516 + ], + [ + -0.00137770373839885, + -0.0031452146358788013, + -0.0015015294775366783 + ], + [ + -0.3751647472381592, + 0.05314668267965317, + -0.28086331486701965 + ], + [ + -0.007756246719509363, + -0.016310883685946465, + -0.02847120724618435 + ], + [ + -0.0002517815155442804, + 0.427451491355896, + 3.640262002591044e-05 + ], + [ + 1.2455408573150635, + -0.0014789876295253634, + 0.0008519256953150034 + ], + [ + 0.004311776254326105, + 0.009671058505773544, + 0.5968337655067444 + ], + [ + 0.1335560381412506, + 0.0011528844479471445, + -0.0008361327927559614 + ], + [ + 0.001167859067209065, + -0.1551152616739273, + 0.00019725598394870758 + ], + [ + -0.0014258474111557007, + 0.0034801543224602938, + 0.0009809854673221707 + ] + ], + [ + [ + -0.047659896314144135, + -0.11130385845899582, + -0.0020901868119835854 + ], + [ + -1.5705475807189941, + -0.0014125468442216516, + -0.0008221857133321464 + ], + [ + -0.16147980093955994, + 0.014729475602507591, + 0.4458121657371521 + ], + [ + -0.00045561062870547175, + -0.1160486489534378, + -0.006125911604613066 + ], + [ + 2.811251401901245, + 0.0016747766640037298, + -0.005349006038159132 + ], + [ + -0.8591147065162659, + 0.0037903853226453066, + 0.00048354381578974426 + ], + [ + 0.006445891689509153, + -0.0036706889513880014, + -0.03472399711608887 + ], + [ + -0.001481462037190795, + 0.0015367366140708327, + -0.0015593112912029028 + ], + [ + -1.5751848220825195, + 0.001112997648306191, + 0.0009848373010754585 + ], + [ + -0.16862420737743378, + -0.016877643764019012, + -0.26229384541511536 + ], + [ + -9.055795817403123e-05, + 0.09453120082616806, + -0.0134742371737957 + ], + [ + 2.811314344406128, + 0.003919574897736311, + 0.005575981922447681 + ], + [ + -0.8299098014831543, + -0.003791244002059102, + 0.0012802339624613523 + ], + [ + 0.005852710455656052, + 0.005849692039191723, + 0.1632416546344757 + ], + [ + -0.0015579514438286424, + 9.288851288147271e-05, + 0.001196552417241037 + ], + [ + 0.00043879495933651924, + 0.04429133981466293, + 0.0002551022043917328 + ], + [ + -0.0019886596128344536, + 0.008745947852730751, + -0.00962099153548479 + ], + [ + 0.5197923183441162, + -0.0010678194230422378, + 0.0002590256044641137 + ], + [ + -0.9051622152328491, + -0.12138096988201141, + 0.25749173760414124 + ], + [ + 0.010689850896596909, + -0.01072163600474596, + 0.20382197201251984 + ], + [ + -0.0009684870601631701, + -0.5894762873649597, + 0.0032688004430383444 + ], + [ + 1.30536949634552, + -0.002206705743446946, + -0.0020471925381571054 + ], + [ + 0.0067055909894406796, + -0.015674468129873276, + -0.9086763262748718 + ], + [ + -0.26612186431884766, + -0.00016191616305150092, + 0.002851327648386359 + ], + [ + 0.003539646975696087, + 0.20451955497264862, + -0.02575569413602352 + ], + [ + 0.003367731347680092, + 0.0018452388467267156, + -0.00026573429931886494 + ], + [ + -0.9464634656906128, + 0.12737642228603363, + -0.2577688992023468 + ], + [ + 0.00046661958913318813, + -0.008693858049809933, + -0.19606870412826538 + ], + [ + -0.0058177076280117035, + 0.6349377036094666, + -0.0003108184027951211 + ], + [ + 1.4694209098815918, + 0.0046353572979569435, + 0.002392316237092018 + ], + [ + 0.022281549870967865, + 0.006433307193219662, + 1.1441218852996826 + ], + [ + -0.16217999160289764, + -0.0005673008854500949, + -0.0028868752997368574 + ], + [ + 0.0011142585426568985, + 0.036793302744627, + 0.06873425096273422 + ], + [ + 0.001964340452104807, + -0.004202086944133043, + 0.0034294212237000465 + ] + ] + ], + "root_positions": [ + [ + 0.014979152008891106, + 0.7896444201469421, + 0.8725281357765198 + ], + [ + 0.12546521425247192, + 0.30551770329475403, + 2.3331315517425537 + ] + ], + "smooth_root_2d": [ + [ + 0.014979152008891106, + 0.8725281357765198 + ], + [ + 0.12546521425247192, + 2.3331315517425537 + ] + ] + }, + { + "type": "right-hand", + "frame_indices": [ + 40, + 155 + ], + "local_joints_rot": [ + [ + [ + -0.20672118663787842, + 0.0004979433142580092, + 0.010066316463053226 + ], + [ + 0.0789145976305008, + 0.0008333905134350061, + -5.267082087812014e-05 + ], + [ + -0.1686924546957016, + -0.0027884345036000013, + 0.0520743690431118 + ], + [ + 0.000989485066384077, + 0.1385614573955536, + 0.0005803265958093107 + ], + [ + 1.0274103879928589, + -0.0004089517460670322, + 0.0007986496202647686 + ], + [ + -0.39034226536750793, + -0.001306047779507935, + -4.922552761854604e-05 + ], + [ + 0.0023066187277436256, + -0.0007853881106711924, + -0.0062883589416742325 + ], + [ + 4.49517356173601e-05, + 0.0033443598076701164, + -0.0014551420463249087 + ], + [ + 0.07268467545509338, + -0.0011258760932832956, + -3.953919076593593e-05 + ], + [ + -0.1719113141298294, + 0.018712127581238747, + 0.06082615628838539 + ], + [ + 0.0011432868195697665, + 0.02744375728070736, + 0.0025501118507236242 + ], + [ + 0.41685307025909424, + -0.002692570211365819, + -0.0006283970433287323 + ], + [ + -0.1283608227968216, + 0.0030534265097230673, + 0.00016949126438703388 + ], + [ + -0.005590266548097134, + 0.0014076301595196128, + -0.038615260273218155 + ], + [ + -0.00013014793512411416, + 0.001360177993774414, + 6.41088408883661e-05 + ], + [ + 0.00010043015936389565, + -0.01370090153068304, + -0.00014910128084011376 + ], + [ + 0.00023336269077844918, + 0.0025421029422432184, + 0.04833226650953293 + ], + [ + 0.056574925780296326, + 0.0006874562823213637, + 0.0004548647266346961 + ], + [ + -0.37481847405433655, + -0.054357241839170456, + 0.2803272306919098 + ], + [ + 0.0013725318713113666, + 0.009074348025023937, + -0.0021504403557628393 + ], + [ + -0.0012184121878817677, + -0.4267229437828064, + 0.011203057132661343 + ], + [ + 1.255251407623291, + 0.0009449978824704885, + 0.0010158077348023653 + ], + [ + -0.003570390399545431, + -0.003947308287024498, + -0.5030224323272705 + ], + [ + 0.1881941556930542, + -0.000495748536195606, + 0.0016725400928407907 + ], + [ + -0.002223622752353549, + 0.11821465194225311, + 0.007546884939074516 + ], + [ + -0.00137770373839885, + -0.0031452146358788013, + -0.0015015294775366783 + ], + [ + -0.3751647472381592, + 0.05314668267965317, + -0.28086331486701965 + ], + [ + -0.007756246719509363, + -0.016310883685946465, + -0.02847120724618435 + ], + [ + -0.0002517815155442804, + 0.427451491355896, + 3.640262002591044e-05 + ], + [ + 1.2455408573150635, + -0.0014789876295253634, + 0.0008519256953150034 + ], + [ + 0.004311776254326105, + 0.009671058505773544, + 0.5968337655067444 + ], + [ + 0.1335560381412506, + 0.0011528844479471445, + -0.0008361327927559614 + ], + [ + 0.001167859067209065, + -0.1551152616739273, + 0.00019725598394870758 + ], + [ + -0.0014258474111557007, + 0.0034801543224602938, + 0.0009809854673221707 + ] + ], + [ + [ + -0.047659896314144135, + -0.11130385845899582, + -0.0020901868119835854 + ], + [ + -1.5705475807189941, + -0.0014125468442216516, + -0.0008221857133321464 + ], + [ + -0.16147980093955994, + 0.014729475602507591, + 0.4458121657371521 + ], + [ + -0.00045561062870547175, + -0.1160486489534378, + -0.006125911604613066 + ], + [ + 2.811251401901245, + 0.0016747766640037298, + -0.005349006038159132 + ], + [ + -0.8591147065162659, + 0.0037903853226453066, + 0.00048354381578974426 + ], + [ + 0.006445891689509153, + -0.0036706889513880014, + -0.03472399711608887 + ], + [ + -0.001481462037190795, + 0.0015367366140708327, + -0.0015593112912029028 + ], + [ + -1.5751848220825195, + 0.001112997648306191, + 0.0009848373010754585 + ], + [ + -0.16862420737743378, + -0.016877643764019012, + -0.26229384541511536 + ], + [ + -9.055795817403123e-05, + 0.09453120082616806, + -0.0134742371737957 + ], + [ + 2.811314344406128, + 0.003919574897736311, + 0.005575981922447681 + ], + [ + -0.8299098014831543, + -0.003791244002059102, + 0.0012802339624613523 + ], + [ + 0.005852710455656052, + 0.005849692039191723, + 0.1632416546344757 + ], + [ + -0.0015579514438286424, + 9.288851288147271e-05, + 0.001196552417241037 + ], + [ + 0.00043879495933651924, + 0.04429133981466293, + 0.0002551022043917328 + ], + [ + -0.0019886596128344536, + 0.008745947852730751, + -0.00962099153548479 + ], + [ + 0.5197923183441162, + -0.0010678194230422378, + 0.0002590256044641137 + ], + [ + -0.9051622152328491, + -0.12138096988201141, + 0.25749173760414124 + ], + [ + 0.010689850896596909, + -0.01072163600474596, + 0.20382197201251984 + ], + [ + -0.0009684870601631701, + -0.5894762873649597, + 0.0032688004430383444 + ], + [ + 1.30536949634552, + -0.002206705743446946, + -0.0020471925381571054 + ], + [ + 0.0067055909894406796, + -0.015674468129873276, + -0.9086763262748718 + ], + [ + -0.26612186431884766, + -0.00016191616305150092, + 0.002851327648386359 + ], + [ + 0.003539646975696087, + 0.20451955497264862, + -0.02575569413602352 + ], + [ + 0.003367731347680092, + 0.0018452388467267156, + -0.00026573429931886494 + ], + [ + -0.9464634656906128, + 0.12737642228603363, + -0.2577688992023468 + ], + [ + 0.00046661958913318813, + -0.008693858049809933, + -0.19606870412826538 + ], + [ + -0.0058177076280117035, + 0.6349377036094666, + -0.0003108184027951211 + ], + [ + 1.4694209098815918, + 0.0046353572979569435, + 0.002392316237092018 + ], + [ + 0.022281549870967865, + 0.006433307193219662, + 1.1441218852996826 + ], + [ + -0.16217999160289764, + -0.0005673008854500949, + -0.0028868752997368574 + ], + [ + 0.0011142585426568985, + 0.036793302744627, + 0.06873425096273422 + ], + [ + 0.001964340452104807, + -0.004202086944133043, + 0.0034294212237000465 + ] + ] + ], + "root_positions": [ + [ + 0.014979152008891106, + 0.7896444201469421, + 0.8725281357765198 + ], + [ + 0.12546521425247192, + 0.30551770329475403, + 2.3331315517425537 + ] + ], + "smooth_root_2d": [ + [ + 0.014979152008891106, + 0.8725281357765198 + ], + [ + 0.12546521425247192, + 2.3331315517425537 + ] + ] + } +] diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/02_multi_text_ee_constraint/meta.json b/kimodo/assets/demo/examples/kimodo-g1-rp/02_multi_text_ee_constraint/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..4503a85e43f3c30c40427efd6ca94c762ddfc3cb --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/02_multi_text_ee_constraint/meta.json @@ -0,0 +1,18 @@ +{ + "texts": [ + "A person walks forward while carrying a box", + "A person sets a box down onto the ground" + ], + "durations": [ + 3.533333333333333, + 4.066666666666666 + ], + "num_samples": 1, + "seed": 60, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 1.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/02_multi_text_ee_constraint/motion.npz b/kimodo/assets/demo/examples/kimodo-g1-rp/02_multi_text_ee_constraint/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..b589b557e8a24213637a9e00dded8dfee82281a7 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/02_multi_text_ee_constraint/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b970313a829200e2d224e8e699b3509ac0a812d1c92aac57c818a0e310b0b13d +size 373806 diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/03_full_body_keyframes/constraints.json b/kimodo/assets/demo/examples/kimodo-g1-rp/03_full_body_keyframes/constraints.json new file mode 100644 index 0000000000000000000000000000000000000000..23868346b262d6726bee170e09fac3bfff7f15b9 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/03_full_body_keyframes/constraints.json @@ -0,0 +1,559 @@ +[ + { + "type": "fullbody", + "frame_indices": [ + 59, + 106, + 148 + ], + "local_joints_rot": [ + [ + [ + 0.42420727014541626, + 0.058721136301755905, + -0.1945635825395584 + ], + [ + -0.5268475413322449, + -0.0005157420528121293, + 0.0004701620200648904 + ], + [ + -0.17267920076847076, + 0.027239520102739334, + 0.36560261249542236 + ], + [ + 0.004160718061029911, + -0.22976335883140564, + 0.010524176992475986 + ], + [ + 1.5825881958007812, + -0.01814083196222782, + -0.00019598894868977368 + ], + [ + -0.8827329277992249, + 0.009902671910822392, + -0.00021610780095215887 + ], + [ + 0.0067768096923828125, + -0.013547217473387718, + -0.16673408448696136 + ], + [ + 0.0006806282908655703, + 0.004601094871759415, + -0.0043960982002317905 + ], + [ + -1.4894901514053345, + -0.003371267579495907, + -0.001970127457752824 + ], + [ + -0.17904962599277496, + 0.004051337484270334, + 0.19225701689720154 + ], + [ + -0.0033012183848768473, + -0.29656991362571716, + 0.004984850063920021 + ], + [ + 1.5931552648544312, + -0.007282367907464504, + -0.0052862209267914295 + ], + [ + -0.35364261269569397, + 0.0049067274667322636, + 0.0010333984391763806 + ], + [ + 0.0023804877419024706, + -0.005421861540526152, + -0.19129839539527893 + ], + [ + 0.0008946731686592102, + 0.0049979668110609055, + -0.0008540445705875754 + ], + [ + -0.00037546976818703115, + -0.09826900064945221, + 0.0006841858848929405 + ], + [ + 0.004415650386363268, + 0.0112489964812994, + 0.025344429537653923 + ], + [ + 0.5182019472122192, + 0.002875699894502759, + 0.002064053900539875 + ], + [ + -0.7899102568626404, + -0.11301380395889282, + 0.261331170797348 + ], + [ + -0.004763631150126457, + 0.003188431030139327, + 0.191846564412117 + ], + [ + -0.0006821855786256492, + -0.24938665330410004, + 0.0013275814708322287 + ], + [ + 1.1367335319519043, + 0.0038948820438236, + 0.0009569167159497738 + ], + [ + 0.006261332891881466, + 0.020894864574074745, + -1.050469160079956 + ], + [ + 0.06118401885032654, + 0.0005131644429638982, + 0.00042430072790011764 + ], + [ + 0.0017778673209249973, + 0.08777552843093872, + -0.044312309473752975 + ], + [ + -0.0006084830965846777, + 0.0022449076641350985, + -0.001873409142717719 + ], + [ + 0.33878403902053833, + -0.04740850627422333, + -0.2796333432197571 + ], + [ + 0.02221747301518917, + 0.013649695552885532, + -0.11847231537103653 + ], + [ + 0.007714178413152695, + 0.6182990074157715, + 0.009067214094102383 + ], + [ + 0.8923805952072144, + -0.00016622581460978836, + 0.0021162345074117184 + ], + [ + 0.0038995807990431786, + -0.006832453887909651, + 0.3025287687778473 + ], + [ + 0.03307999297976494, + 0.0005516205565072596, + 0.0009820020059123635 + ], + [ + 0.0015379488468170166, + -0.08221427351236343, + -0.014401843771338463 + ], + [ + -0.00022057670867070556, + 0.002010792726650834, + 0.0012923656031489372 + ] + ], + [ + [ + -0.08197958767414093, + 0.10326994955539703, + -0.1510602980852127 + ], + [ + 0.28157129883766174, + 0.0011461800895631313, + 0.000703590689226985 + ], + [ + -0.182321235537529, + 0.05269569158554077, + 0.2730983793735504 + ], + [ + -0.0003947282093577087, + 0.09641454368829727, + 0.0040251282043755054 + ], + [ + 1.089223861694336, + -0.00700604822486639, + -0.002539312234148383 + ], + [ + -0.09248486906290054, + 0.003849609522148967, + 0.0016473153373226523 + ], + [ + -0.010541710071265697, + 0.004344945307821035, + 0.07663393765687943 + ], + [ + -0.00044715296826325357, + -0.004340745974332094, + 0.007171581499278545 + ], + [ + -0.3379390239715576, + 0.0015806800220161676, + -0.0003471111413091421 + ], + [ + -0.1781967729330063, + 0.016616491600871086, + 0.1652776598930359 + ], + [ + -0.002019439358264208, + -0.11581386625766754, + 0.0009603232610970736 + ], + [ + 0.6794841289520264, + -5.403390241554007e-05, + -0.0012657493352890015 + ], + [ + -0.09013757854700089, + 0.0018549489323049784, + -0.000238976048422046 + ], + [ + -0.0009166855015791953, + -0.0007138565997593105, + -0.0742788091301918 + ], + [ + -0.0009655999601818621, + 0.0029521933756768703, + -0.00039851426845416427 + ], + [ + -0.0006129079265519977, + -0.19495022296905518, + -0.0019512351136654615 + ], + [ + 0.0019297772087156773, + -0.0025066917296499014, + 0.1518552601337433 + ], + [ + 0.18073193728923798, + -0.0008597049745731056, + 0.00023304206843022257 + ], + [ + -0.19048453867435455, + -0.02173178642988205, + 0.2785468101501465 + ], + [ + 0.0032724339980632067, + 0.001481848070397973, + 0.00837984960526228 + ], + [ + 0.0037242062389850616, + -0.19455766677856445, + 0.009616612456738949 + ], + [ + -0.19767794013023376, + 0.004192049615085125, + 0.004219892434775829 + ], + [ + -0.018522148951888084, + 0.01758752018213272, + -1.4997444152832031 + ], + [ + -0.07066819816827774, + -0.0006776255904696882, + 0.00122307357378304 + ], + [ + 0.007704276591539383, + 0.14503517746925354, + 0.0951184555888176 + ], + [ + 0.004533262457698584, + -0.0066575342789292336, + -0.010643035173416138 + ], + [ + 0.3773331642150879, + -0.05414784327149391, + -0.2780730128288269 + ], + [ + 0.003753547091037035, + 0.002539943205192685, + 0.12321871519088745 + ], + [ + -0.004724413156509399, + 0.46992960572242737, + 0.001832474721595645 + ], + [ + 1.2976007461547852, + 0.0007234009681269526, + -0.001626322278752923 + ], + [ + -0.0016050372505560517, + -0.00880438182502985, + 0.17947044968605042 + ], + [ + 0.05334911122918129, + -0.00018671243742574006, + 0.0010833276901394129 + ], + [ + -0.0015367609448730946, + -0.05425700917840004, + 0.01668459363281727 + ], + [ + -0.00021225935779511929, + 0.001713683595880866, + 0.0009809889597818255 + ] + ], + [ + [ + -0.21817633509635925, + -0.012708673253655434, + -0.029821090400218964 + ], + [ + 0.3743710219860077, + 0.0007941523799672723, + 0.00032366320374421775 + ], + [ + -0.16750676929950714, + 0.003018906805664301, + 0.07928019016981125 + ], + [ + -0.0003895726113114506, + 0.030501781031489372, + 0.0013912678696215153 + ], + [ + 0.2578306794166565, + -0.0026517061050981283, + -0.0001139347514254041 + ], + [ + -0.227533221244812, + 0.0004564583650790155, + -0.0004175934591330588 + ], + [ + -0.0015815469669178128, + 0.0026496825739741325, + -0.017801448702812195 + ], + [ + 0.00024288007989525795, + 0.000392801477573812, + -2.9845070457668044e-05 + ], + [ + 0.31938642263412476, + -0.0006790655897930264, + -0.0004276619874872267 + ], + [ + -0.17199693620204926, + 0.007707139942795038, + 0.014987054280936718 + ], + [ + 0.0012992072151973844, + 0.003620905103161931, + -0.001210421440191567 + ], + [ + 0.22853288054466248, + -0.0018797506345435977, + -0.0002660619793459773 + ], + [ + -0.1335543692111969, + 0.0010313205420970917, + 0.0001083972238120623 + ], + [ + 0.003520265920087695, + 0.0035283963661640882, + 0.016698163002729416 + ], + [ + 0.0001443400833522901, + -0.001745356246829033, + -2.3336755475611426e-05 + ], + [ + 0.0003554633294697851, + -0.05629483610391617, + -0.0006463310564868152 + ], + [ + -0.00298635708168149, + 0.0020182463340461254, + -0.03614736720919609 + ], + [ + 0.21955031156539917, + 0.0005465149879455566, + 0.00011243963672313839 + ], + [ + -0.0715053528547287, + -0.010282701812684536, + 0.28057143092155457 + ], + [ + 0.0007245761225931346, + 0.0019379559671506286, + -0.018530432134866714 + ], + [ + -0.0020012110471725464, + -0.5585712194442749, + 0.0002525273594073951 + ], + [ + 1.1451164484024048, + 0.000756395107600838, + -0.00042264885269105434 + ], + [ + -0.004087591078132391, + -0.0022635578643530607, + -0.1811828911304474 + ], + [ + 0.15393203496932983, + -0.00010327681229682639, + 0.000951180059928447 + ], + [ + -0.0005707733216695487, + 0.07005079090595245, + -0.0018504050094634295 + ], + [ + -0.0013123765820637345, + -0.0004375300486572087, + 0.0002970081695821136 + ], + [ + -0.09115279465913773, + 0.013008617796003819, + -0.2808595299720764 + ], + [ + 0.0015214721206575632, + -0.007811791729182005, + 0.031220799311995506 + ], + [ + -0.00048553026863373816, + 0.5777612328529358, + 0.0003351669874973595 + ], + [ + 1.0913182497024536, + 0.0011191898956894875, + -0.0027903772424906492 + ], + [ + 0.000775794149376452, + 0.00010774911061162129, + 0.10287072509527206 + ], + [ + 0.0997936949133873, + 0.0003015398688148707, + -0.0006937433499842882 + ], + [ + 0.0003619014751166105, + -0.18787385523319244, + -0.0010270585771650076 + ], + [ + -0.001584835583344102, + 0.0037561857607215643, + -0.002414965769276023 + ] + ] + ], + "root_positions": [ + [ + -0.17535515129566193, + 0.5689253807067871, + 0.9417929649353027 + ], + [ + -0.16934014856815338, + 0.7382326722145081, + 2.169330596923828 + ], + [ + -0.1823902279138565, + 0.7819305658340454, + 2.954490900039673 + ] + ], + "smooth_root_2d": [ + [ + -0.17535515129566193, + 0.9417929649353027 + ], + [ + -0.16934014856815338, + 2.169330596923828 + ], + [ + -0.1823902279138565, + 2.954490900039673 + ] + ] + } +] diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/03_full_body_keyframes/meta.json b/kimodo/assets/demo/examples/kimodo-g1-rp/03_full_body_keyframes/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..fc2bf5c73be9c41e967619bfdbd7f22478074926 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/03_full_body_keyframes/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person walking forward picks up something off the ground", + "duration": 5.0, + "num_samples": 1, + "seed": 51, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 1.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/03_full_body_keyframes/motion.npz b/kimodo/assets/demo/examples/kimodo-g1-rp/03_full_body_keyframes/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..60d3a87ab762ec3d073c2b78d3678761dfdc9eeb --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/03_full_body_keyframes/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4714a70db9e5ec4b4875ae6163d4161192b778029be263e033bfb68e61dfbb56 +size 246198 diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/04_ee_constraint/constraints.json b/kimodo/assets/demo/examples/kimodo-g1-rp/04_ee_constraint/constraints.json new file mode 100644 index 0000000000000000000000000000000000000000..2635055f4ca517ec69b58dcb569b0eb9f2c68fcd --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/04_ee_constraint/constraints.json @@ -0,0 +1,1320 @@ +[ + { + "type": "right-hand", + "frame_indices": [ + 129, + 93, + 0 + ], + "local_joints_rot": [ + [ + [ + -0.11223886162042618, + 0.20531758666038513, + 0.13568778336048126 + ], + [ + 0.1075688898563385, + 0.0032202948350459337, + 0.0006892754463478923 + ], + [ + -0.17058254778385162, + -0.011657492257654667, + -0.23103317618370056 + ], + [ + -0.02866872400045395, + 0.4262913167476654, + -0.010209682397544384 + ], + [ + 0.2924644649028778, + 0.007188746705651283, + 0.0005000674282200634 + ], + [ + -0.13080132007598877, + -0.0029640060383826494, + -0.0007075564353726804 + ], + [ + -0.005761375650763512, + -0.002191383158788085, + 0.15397773683071136 + ], + [ + -0.00023041102394927293, + -0.0010889451950788498, + 0.0007837787852622569 + ], + [ + -0.3537895977497101, + -0.0006238390924409032, + -0.0010272490326315165 + ], + [ + -0.16032733023166656, + -0.02506295032799244, + -0.22620464861392975 + ], + [ + 0.0138308797031641, + 0.21655774116516113, + 0.0317748561501503 + ], + [ + 1.5745534896850586, + 0.003732866607606411, + 0.0021063678432255983 + ], + [ + -0.17066748440265656, + -0.002285068854689598, + -0.0029538189992308617 + ], + [ + 0.02313886024057865, + -0.07020875811576843, + -0.05658446252346039 + ], + [ + 2.5580025976523757e-05, + 0.004435115493834019, + -0.006514436099678278 + ], + [ + 0.0015886364271864295, + -0.292732834815979, + -0.0014166575856506824 + ], + [ + -0.008558829315006733, + 0.0066470191814005375, + -0.010221566073596478 + ], + [ + 0.10141321271657944, + -0.0028386565390974283, + -0.0006978976307436824 + ], + [ + 0.002506372518837452, + 0.001101600006222725, + 0.2779805362224579 + ], + [ + 0.020367039367556572, + -0.028616085648536682, + 0.0971180647611618 + ], + [ + -0.011572631075978279, + -0.5930124521255493, + -0.026975814253091812 + ], + [ + 0.9286840558052063, + -0.00046807233593426645, + -0.00013331411173567176 + ], + [ + -0.031172338873147964, + -0.04484722763299942, + 0.03643424063920975 + ], + [ + 0.03150894120335579, + -0.00101278827060014, + 0.0015338404336944222 + ], + [ + 0.0005915925721637905, + 0.0930531769990921, + -0.028835415840148926 + ], + [ + -0.001440802589058876, + 0.0010614224011078477, + 0.0006542576011270285 + ], + [ + -0.4149414598941803, + 0.06656259298324585, + -0.2730332314968109 + ], + [ + -0.006371266208589077, + -0.02150307223200798, + -1.3590242862701416 + ], + [ + 0.00956002902239561, + -0.17155548930168152, + 0.026624836027622223 + ], + [ + 0.8084958791732788, + -0.003991501871496439, + 0.0007233448559418321 + ], + [ + -0.020737944170832634, + -0.011397535912692547, + 0.14019189774990082 + ], + [ + -0.18261606991291046, + 0.005134414881467819, + -0.001045998651534319 + ], + [ + -0.028427572920918465, + -0.6557883620262146, + 0.038063470274209976 + ], + [ + 0.005555277690291405, + 0.012246276251971722, + 0.004770371131598949 + ] + ], + [ + [ + -0.06392758339643478, + 0.3478183448314667, + 0.1171446293592453 + ], + [ + 0.12243298441171646, + 0.003146131755784154, + 0.00017438907525502145 + ], + [ + -0.17841783165931702, + -0.0256511103361845, + -0.2805330455303192 + ], + [ + -0.022625330835580826, + 0.348234087228775, + -0.009928824380040169 + ], + [ + 0.28284141421318054, + 0.009495020844042301, + 0.0010556986089795828 + ], + [ + -0.17478667199611664, + -0.004891794174909592, + -0.0013969563879072666 + ], + [ + -0.002641322324052453, + -0.005833400413393974, + 0.20226475596427917 + ], + [ + -0.0009078677394427359, + -0.002073301700875163, + 0.0012749496381729841 + ], + [ + -0.48070675134658813, + 0.0005347213009372354, + -0.0004243548901285976 + ], + [ + -0.16694584488868713, + -0.03390314802527428, + -0.09055406600236893 + ], + [ + 0.009182179346680641, + 0.1743844896554947, + 0.01932411640882492 + ], + [ + 1.6481772661209106, + 0.0002097517135553062, + 0.0010239556431770325 + ], + [ + -0.17133140563964844, + 0.0028362423181533813, + -0.004689408931881189 + ], + [ + 0.025385459885001183, + -0.06771048158407211, + -0.011561849154531956 + ], + [ + -0.00012663791130762547, + 0.001872184220701456, + -0.002834505634382367 + ], + [ + 0.001523697399534285, + -0.48211750388145447, + -0.0005278618773445487 + ], + [ + -0.00822246354073286, + -0.00923906546086073, + -0.01643195189535618 + ], + [ + 0.04035002365708351, + -0.004922393709421158, + -0.0005214703269302845 + ], + [ + -0.02120170183479786, + -0.000465662480564788, + 0.27964550256729126 + ], + [ + 0.042349521070718765, + -0.043123405426740646, + 0.21025802195072174 + ], + [ + -0.01620035618543625, + -0.5838293433189392, + -0.03403719887137413 + ], + [ + 1.1832103729248047, + 0.0004754749243147671, + -0.0014872060855850577 + ], + [ + -0.040768858045339584, + -0.04618615657091141, + 0.04847611486911774 + ], + [ + 0.04482508823275566, + -0.0005392982857301831, + 0.00035259113064967096 + ], + [ + 0.00015537742001470178, + -0.024237608537077904, + -0.003044326091185212 + ], + [ + -0.0012453795643523335, + 0.004743263591080904, + 0.004625802394002676 + ], + [ + -0.14595142006874084, + 0.0308919008821249, + -0.2779163420200348 + ], + [ + -0.03314027562737465, + -0.07205720245838165, + -1.3401029109954834 + ], + [ + 0.02448190003633499, + -0.468079537153244, + 0.018310735002160072 + ], + [ + 0.9222347140312195, + -0.00624655419960618, + -0.0003706512216012925 + ], + [ + 0.0311859343200922, + -0.01980999857187271, + -0.4311404228210449 + ], + [ + -0.05887744575738907, + 0.009565972723066807, + 0.0008855919586494565 + ], + [ + -0.0638674795627594, + -1.1882448196411133, + -0.07744041085243225 + ], + [ + 0.002320833969861269, + 0.014880148693919182, + 0.00827236007899046 + ] + ], + [ + [ + 0.028708748519420624, + 0.023731501772999763, + -0.05906220152974129 + ], + [ + 0.36697518825531006, + 0.0020822372753173113, + 9.442192094866186e-06 + ], + [ + -0.17328320443630219, + -0.029694421216845512, + -0.2592017650604248 + ], + [ + -0.027558816596865654, + 0.44522055983543396, + 0.00263651879504323 + ], + [ + 0.45747342705726624, + 0.006375299766659737, + 0.000838644162286073 + ], + [ + -0.29932498931884766, + -0.0034287264570593834, + -0.005712746176868677 + ], + [ + 0.010242770425975323, + 0.0686849057674408, + 0.12300582230091095 + ], + [ + 0.0019906593952327967, + -0.006487288512289524, + 0.004740884527564049 + ], + [ + -0.417245090007782, + 0.002172173699364066, + -0.000527464144397527 + ], + [ + -0.16229933500289917, + -0.015825729817152023, + 0.26093363761901855 + ], + [ + -0.01547759398818016, + -0.4560239017009735, + -0.001296655391342938 + ], + [ + 0.520811140537262, + -0.016100304201245308, + -0.0033653294667601585 + ], + [ + -0.061035193502902985, + 0.013747301883995533, + 0.0011975782690569758 + ], + [ + 0.002211581217125058, + 0.013100380077958107, + -0.41168421506881714 + ], + [ + 0.000723487522918731, + -0.0009448538185097277, + -0.0023157261312007904 + ], + [ + -0.0008414603653363883, + -0.22904154658317566, + 0.0037871438544243574 + ], + [ + -0.004434449132531881, + -0.0019493037834763527, + 0.04349867254495621 + ], + [ + 0.11113234609365463, + -0.001496539101935923, + -6.745033260813216e-06 + ], + [ + 0.03568394109606743, + 0.00850191805511713, + 0.2815527021884918 + ], + [ + 0.007574420887976885, + -0.005988026969134808, + -0.04585442319512367 + ], + [ + -0.014899174682796001, + -0.6360949277877808, + 0.014495083130896091 + ], + [ + 1.1318601369857788, + -0.0009174949955195189, + -0.008180576376616955 + ], + [ + -0.038145799189805984, + -0.05923198536038399, + -0.04122990742325783 + ], + [ + 0.07719366252422333, + -0.0010574767366051674, + 0.0009220906649716198 + ], + [ + -0.0010063578374683857, + 0.12876589596271515, + -0.021289559081196785 + ], + [ + -0.0008511252817697823, + -0.0003541657351888716, + -0.0006832815706729889 + ], + [ + 0.10374817997217178, + -0.014772959984838963, + -0.28235113620758057 + ], + [ + 0.029763568192720413, + 0.00017807059339247644, + 0.007343007251620293 + ], + [ + -0.0064206854440271854, + 0.3665950298309326, + -0.0003897137939929962 + ], + [ + 1.0820642709732056, + -0.0005379249923862517, + -0.0039028781466186047 + ], + [ + -0.004170380067080259, + 0.06480656564235687, + -0.10721305757761002 + ], + [ + -0.06350508332252502, + 0.0011865347623825073, + -0.0005369586870074272 + ], + [ + -0.0021817537490278482, + -0.08756759762763977, + -0.008148521184921265 + ], + [ + 0.00243115471675992, + -0.003949992824345827, + 0.005949904676526785 + ] + ] + ], + "root_positions": [ + [ + 2.639763593673706, + 0.767427384853363, + 2.341259479522705 + ], + [ + 1.9461809396743774, + 0.7706995010375977, + 1.7243560552597046 + ], + [ + 0.003315839683637023, + 0.7415399551391602, + -0.0012030001962557435 + ] + ], + "smooth_root_2d": [ + [ + 2.639763593673706, + 2.341259479522705 + ], + [ + 1.9461809396743774, + 1.7243560552597046 + ], + [ + 0.003315839683637023, + -0.0012030001962557435 + ] + ] + }, + { + "type": "left-foot", + "frame_indices": [ + 93, + 0 + ], + "local_joints_rot": [ + [ + [ + -0.06392758339643478, + 0.3478183448314667, + 0.1171446293592453 + ], + [ + 0.12243298441171646, + 0.003146131755784154, + 0.00017438907525502145 + ], + [ + -0.17841783165931702, + -0.0256511103361845, + -0.2805330455303192 + ], + [ + -0.022625330835580826, + 0.348234087228775, + -0.009928824380040169 + ], + [ + 0.28284141421318054, + 0.009495020844042301, + 0.0010556986089795828 + ], + [ + -0.17478667199611664, + -0.004891794174909592, + -0.0013969563879072666 + ], + [ + -0.002641322324052453, + -0.005833400413393974, + 0.20226475596427917 + ], + [ + -0.0009078677394427359, + -0.002073301700875163, + 0.0012749496381729841 + ], + [ + -0.48070675134658813, + 0.0005347213009372354, + -0.0004243548901285976 + ], + [ + -0.16694584488868713, + -0.03390314802527428, + -0.09055406600236893 + ], + [ + 0.009182179346680641, + 0.1743844896554947, + 0.01932411640882492 + ], + [ + 1.6481772661209106, + 0.0002097517135553062, + 0.0010239556431770325 + ], + [ + -0.17133140563964844, + 0.0028362423181533813, + -0.004689408931881189 + ], + [ + 0.025385459885001183, + -0.06771048158407211, + -0.011561849154531956 + ], + [ + -0.00012663791130762547, + 0.001872184220701456, + -0.002834505634382367 + ], + [ + 0.001523697399534285, + -0.48211750388145447, + -0.0005278618773445487 + ], + [ + -0.00822246354073286, + -0.00923906546086073, + -0.01643195189535618 + ], + [ + 0.04035002365708351, + -0.004922393709421158, + -0.0005214703269302845 + ], + [ + -0.02120170183479786, + -0.000465662480564788, + 0.27964550256729126 + ], + [ + 0.042349521070718765, + -0.043123405426740646, + 0.21025802195072174 + ], + [ + -0.01620035618543625, + -0.5838293433189392, + -0.03403719887137413 + ], + [ + 1.1832103729248047, + 0.0004754749243147671, + -0.0014872060855850577 + ], + [ + -0.040768858045339584, + -0.04618615657091141, + 0.04847611486911774 + ], + [ + 0.04482508823275566, + -0.0005392982857301831, + 0.00035259113064967096 + ], + [ + 0.00015537742001470178, + -0.024237608537077904, + -0.003044326091185212 + ], + [ + -0.0012453795643523335, + 0.004743263591080904, + 0.004625802394002676 + ], + [ + -0.14595142006874084, + 0.0308919008821249, + -0.2779163420200348 + ], + [ + -0.03314027562737465, + -0.07205720245838165, + -1.3401029109954834 + ], + [ + 0.02448190003633499, + -0.468079537153244, + 0.018310735002160072 + ], + [ + 0.9222347140312195, + -0.00624655419960618, + -0.0003706512216012925 + ], + [ + 0.0311859343200922, + -0.01980999857187271, + -0.4311404228210449 + ], + [ + -0.05887744575738907, + 0.009565972723066807, + 0.0008855919586494565 + ], + [ + -0.0638674795627594, + -1.1882448196411133, + -0.07744041085243225 + ], + [ + 0.002320833969861269, + 0.014880148693919182, + 0.00827236007899046 + ] + ], + [ + [ + 0.028708748519420624, + 0.023731501772999763, + -0.05906220152974129 + ], + [ + 0.36697518825531006, + 0.0020822372753173113, + 9.442192094866186e-06 + ], + [ + -0.17328320443630219, + -0.029694421216845512, + -0.2592017650604248 + ], + [ + -0.027558816596865654, + 0.44522055983543396, + 0.00263651879504323 + ], + [ + 0.45747342705726624, + 0.006375299766659737, + 0.000838644162286073 + ], + [ + -0.29932498931884766, + -0.0034287264570593834, + -0.005712746176868677 + ], + [ + 0.010242770425975323, + 0.0686849057674408, + 0.12300582230091095 + ], + [ + 0.0019906593952327967, + -0.006487288512289524, + 0.004740884527564049 + ], + [ + -0.417245090007782, + 0.002172173699364066, + -0.000527464144397527 + ], + [ + -0.16229933500289917, + -0.015825729817152023, + 0.26093363761901855 + ], + [ + -0.01547759398818016, + -0.4560239017009735, + -0.001296655391342938 + ], + [ + 0.520811140537262, + -0.016100304201245308, + -0.0033653294667601585 + ], + [ + -0.061035193502902985, + 0.013747301883995533, + 0.0011975782690569758 + ], + [ + 0.002211581217125058, + 0.013100380077958107, + -0.41168421506881714 + ], + [ + 0.000723487522918731, + -0.0009448538185097277, + -0.0023157261312007904 + ], + [ + -0.0008414603653363883, + -0.22904154658317566, + 0.0037871438544243574 + ], + [ + -0.004434449132531881, + -0.0019493037834763527, + 0.04349867254495621 + ], + [ + 0.11113234609365463, + -0.001496539101935923, + -6.745033260813216e-06 + ], + [ + 0.03568394109606743, + 0.00850191805511713, + 0.2815527021884918 + ], + [ + 0.007574420887976885, + -0.005988026969134808, + -0.04585442319512367 + ], + [ + -0.014899174682796001, + -0.6360949277877808, + 0.014495083130896091 + ], + [ + 1.1318601369857788, + -0.0009174949955195189, + -0.008180576376616955 + ], + [ + -0.038145799189805984, + -0.05923198536038399, + -0.04122990742325783 + ], + [ + 0.07719366252422333, + -0.0010574767366051674, + 0.0009220906649716198 + ], + [ + -0.0010063578374683857, + 0.12876589596271515, + -0.021289559081196785 + ], + [ + -0.0008511252817697823, + -0.0003541657351888716, + -0.0006832815706729889 + ], + [ + 0.10374817997217178, + -0.014772959984838963, + -0.28235113620758057 + ], + [ + 0.029763568192720413, + 0.00017807059339247644, + 0.007343007251620293 + ], + [ + -0.0064206854440271854, + 0.3665950298309326, + -0.0003897137939929962 + ], + [ + 1.0820642709732056, + -0.0005379249923862517, + -0.0039028781466186047 + ], + [ + -0.004170380067080259, + 0.06480656564235687, + -0.10721305757761002 + ], + [ + -0.06350508332252502, + 0.0011865347623825073, + -0.0005369586870074272 + ], + [ + -0.0021817537490278482, + -0.08756759762763977, + -0.008148521184921265 + ], + [ + 0.00243115471675992, + -0.003949992824345827, + 0.005949904676526785 + ] + ] + ], + "root_positions": [ + [ + 1.9461809396743774, + 0.7706995010375977, + 1.7243560552597046 + ], + [ + 0.003315839683637023, + 0.7415399551391602, + -0.0012030001962557435 + ] + ], + "smooth_root_2d": [ + [ + 1.9461809396743774, + 1.7243560552597046 + ], + [ + 0.003315839683637023, + -0.0012030001962557435 + ] + ] + }, + { + "type": "right-foot", + "frame_indices": [ + 0 + ], + "local_joints_rot": [ + [ + [ + 0.028708748519420624, + 0.023731501772999763, + -0.05906220152974129 + ], + [ + 0.36697518825531006, + 0.0020822372753173113, + 9.442192094866186e-06 + ], + [ + -0.17328320443630219, + -0.029694421216845512, + -0.2592017650604248 + ], + [ + -0.027558816596865654, + 0.44522055983543396, + 0.00263651879504323 + ], + [ + 0.45747342705726624, + 0.006375299766659737, + 0.000838644162286073 + ], + [ + -0.29932498931884766, + -0.0034287264570593834, + -0.005712746176868677 + ], + [ + 0.010242770425975323, + 0.0686849057674408, + 0.12300582230091095 + ], + [ + 0.0019906593952327967, + -0.006487288512289524, + 0.004740884527564049 + ], + [ + -0.417245090007782, + 0.002172173699364066, + -0.000527464144397527 + ], + [ + -0.16229933500289917, + -0.015825729817152023, + 0.26093363761901855 + ], + [ + -0.01547759398818016, + -0.4560239017009735, + -0.001296655391342938 + ], + [ + 0.520811140537262, + -0.016100304201245308, + -0.0033653294667601585 + ], + [ + -0.061035193502902985, + 0.013747301883995533, + 0.0011975782690569758 + ], + [ + 0.002211581217125058, + 0.013100380077958107, + -0.41168421506881714 + ], + [ + 0.000723487522918731, + -0.0009448538185097277, + -0.0023157261312007904 + ], + [ + -0.0008414603653363883, + -0.22904154658317566, + 0.0037871438544243574 + ], + [ + -0.004434449132531881, + -0.0019493037834763527, + 0.04349867254495621 + ], + [ + 0.11113234609365463, + -0.001496539101935923, + -6.745033260813216e-06 + ], + [ + 0.03568394109606743, + 0.00850191805511713, + 0.2815527021884918 + ], + [ + 0.007574420887976885, + -0.005988026969134808, + -0.04585442319512367 + ], + [ + -0.014899174682796001, + -0.6360949277877808, + 0.014495083130896091 + ], + [ + 1.1318601369857788, + -0.0009174949955195189, + -0.008180576376616955 + ], + [ + -0.038145799189805984, + -0.05923198536038399, + -0.04122990742325783 + ], + [ + 0.07719366252422333, + -0.0010574767366051674, + 0.0009220906649716198 + ], + [ + -0.0010063578374683857, + 0.12876589596271515, + -0.021289559081196785 + ], + [ + -0.0008511252817697823, + -0.0003541657351888716, + -0.0006832815706729889 + ], + [ + 0.10374817997217178, + -0.014772959984838963, + -0.28235113620758057 + ], + [ + 0.029763568192720413, + 0.00017807059339247644, + 0.007343007251620293 + ], + [ + -0.0064206854440271854, + 0.3665950298309326, + -0.0003897137939929962 + ], + [ + 1.0820642709732056, + -0.0005379249923862517, + -0.0039028781466186047 + ], + [ + -0.004170380067080259, + 0.06480656564235687, + -0.10721305757761002 + ], + [ + -0.06350508332252502, + 0.0011865347623825073, + -0.0005369586870074272 + ], + [ + -0.0021817537490278482, + -0.08756759762763977, + -0.008148521184921265 + ], + [ + 0.00243115471675992, + -0.003949992824345827, + 0.005949904676526785 + ] + ] + ], + "root_positions": [ + [ + 0.003315839683637023, + 0.7415399551391602, + -0.0012030001962557435 + ] + ], + "smooth_root_2d": [ + [ + 0.003315839683637023, + -0.0012030001962557435 + ] + ] + }, + { + "type": "left-hand", + "frame_indices": [ + 0 + ], + "local_joints_rot": [ + [ + [ + 0.028708748519420624, + 0.023731501772999763, + -0.05906220152974129 + ], + [ + 0.36697518825531006, + 0.0020822372753173113, + 9.442192094866186e-06 + ], + [ + -0.17328320443630219, + -0.029694421216845512, + -0.2592017650604248 + ], + [ + -0.027558816596865654, + 0.44522055983543396, + 0.00263651879504323 + ], + [ + 0.45747342705726624, + 0.006375299766659737, + 0.000838644162286073 + ], + [ + -0.29932498931884766, + -0.0034287264570593834, + -0.005712746176868677 + ], + [ + 0.010242770425975323, + 0.0686849057674408, + 0.12300582230091095 + ], + [ + 0.0019906593952327967, + -0.006487288512289524, + 0.004740884527564049 + ], + [ + -0.417245090007782, + 0.002172173699364066, + -0.000527464144397527 + ], + [ + -0.16229933500289917, + -0.015825729817152023, + 0.26093363761901855 + ], + [ + -0.01547759398818016, + -0.4560239017009735, + -0.001296655391342938 + ], + [ + 0.520811140537262, + -0.016100304201245308, + -0.0033653294667601585 + ], + [ + -0.061035193502902985, + 0.013747301883995533, + 0.0011975782690569758 + ], + [ + 0.002211581217125058, + 0.013100380077958107, + -0.41168421506881714 + ], + [ + 0.000723487522918731, + -0.0009448538185097277, + -0.0023157261312007904 + ], + [ + -0.0008414603653363883, + -0.22904154658317566, + 0.0037871438544243574 + ], + [ + -0.004434449132531881, + -0.0019493037834763527, + 0.04349867254495621 + ], + [ + 0.11113234609365463, + -0.001496539101935923, + -6.745033260813216e-06 + ], + [ + 0.03568394109606743, + 0.00850191805511713, + 0.2815527021884918 + ], + [ + 0.007574420887976885, + -0.005988026969134808, + -0.04585442319512367 + ], + [ + -0.014899174682796001, + -0.6360949277877808, + 0.014495083130896091 + ], + [ + 1.1318601369857788, + -0.0009174949955195189, + -0.008180576376616955 + ], + [ + -0.038145799189805984, + -0.05923198536038399, + -0.04122990742325783 + ], + [ + 0.07719366252422333, + -0.0010574767366051674, + 0.0009220906649716198 + ], + [ + -0.0010063578374683857, + 0.12876589596271515, + -0.021289559081196785 + ], + [ + -0.0008511252817697823, + -0.0003541657351888716, + -0.0006832815706729889 + ], + [ + 0.10374817997217178, + -0.014772959984838963, + -0.28235113620758057 + ], + [ + 0.029763568192720413, + 0.00017807059339247644, + 0.007343007251620293 + ], + [ + -0.0064206854440271854, + 0.3665950298309326, + -0.0003897137939929962 + ], + [ + 1.0820642709732056, + -0.0005379249923862517, + -0.0039028781466186047 + ], + [ + -0.004170380067080259, + 0.06480656564235687, + -0.10721305757761002 + ], + [ + -0.06350508332252502, + 0.0011865347623825073, + -0.0005369586870074272 + ], + [ + -0.0021817537490278482, + -0.08756759762763977, + -0.008148521184921265 + ], + [ + 0.00243115471675992, + -0.003949992824345827, + 0.005949904676526785 + ] + ] + ], + "root_positions": [ + [ + 0.003315839683637023, + 0.7415399551391602, + -0.0012030001962557435 + ] + ], + "smooth_root_2d": [ + [ + 0.003315839683637023, + -0.0012030001962557435 + ] + ] + } +] diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/04_ee_constraint/meta.json b/kimodo/assets/demo/examples/kimodo-g1-rp/04_ee_constraint/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..129caa3add80d763f4b0b240f4ce4f7016cdea1c --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/04_ee_constraint/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person walks diagonally to the left and waves at someone on their right", + "duration": 4.966666666666667, + "num_samples": 1, + "seed": 44, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/04_ee_constraint/motion.npz b/kimodo/assets/demo/examples/kimodo-g1-rp/04_ee_constraint/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..fdcd9a3e6dd49e62aed412758c1abbb3b4123fa4 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/04_ee_constraint/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cad86faf99cd4d951371622ab3eba097a6210fc18950dadd1da578dc1e7cd73 +size 244562 diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/05_root_path/constraints.json b/kimodo/assets/demo/examples/kimodo-g1-rp/05_root_path/constraints.json new file mode 100644 index 0000000000000000000000000000000000000000..681efcc539c7635aea937ed62137fda58fc9b5f7 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/05_root_path/constraints.json @@ -0,0 +1,914 @@ +[ + { + "type": "root2d", + "frame_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180 + ], + "smooth_root_2d": [ + [ + -0.024789854884147644, + 0.01764228567481041 + ], + [ + -0.019911596551537514, + 0.03666473180055618 + ], + [ + -0.015032900497317314, + 0.05568705126643181 + ], + [ + -0.010153300128877163, + 0.07470902800559998 + ], + [ + -0.005272198934108019, + 0.09373034536838531 + ], + [ + -0.00038888092967681587, + 0.11275061219930649 + ], + [ + 0.004497467540204525, + 0.1317693293094635 + ], + [ + 0.009387745521962643, + 0.15078598260879517 + ], + [ + 0.014282921329140663, + 0.16979998350143433 + ], + [ + 0.019184017553925514, + 0.18881070613861084 + ], + [ + 0.024092093110084534, + 0.20781749486923218 + ], + [ + 0.029008235782384872, + 0.226819708943367 + ], + [ + 0.033933546394109726, + 0.24581670761108398 + ], + [ + 0.038869116455316544, + 0.2648078203201294 + ], + [ + 0.04381602630019188, + 0.2837924659252167 + ], + [ + 0.048775337636470795, + 0.30277004837989807 + ], + [ + 0.05374806746840477, + 0.321740061044693 + ], + [ + 0.058735184371471405, + 0.3407020568847656 + ], + [ + 0.06373759359121323, + 0.35965561866760254 + ], + [ + 0.06875615566968918, + 0.37860047817230225 + ], + [ + 0.07379162311553955, + 0.3975364565849304 + ], + [ + 0.07884468138217926, + 0.4164634943008423 + ], + [ + 0.08391592651605606, + 0.43538162112236023 + ], + [ + 0.08900584280490875, + 0.45429113507270813 + ], + [ + 0.09411482512950897, + 0.47319236397743225 + ], + [ + 0.0992431491613388, + 0.49208587408065796 + ], + [ + 0.10439097136259079, + 0.5109724998474121 + ], + [ + 0.1095583438873291, + 0.5298531651496887 + ], + [ + 0.11474518477916718, + 0.5487290620803833 + ], + [ + 0.11995130032300949, + 0.5676016807556152 + ], + [ + 0.12517637014389038, + 0.5864726901054382 + ], + [ + 0.13041996955871582, + 0.6053440570831299 + ], + [ + 0.13568153977394104, + 0.6242179274559021 + ], + [ + 0.1409604400396347, + 0.6430967450141907 + ], + [ + 0.14625589549541473, + 0.6619831919670105 + ], + [ + 0.15156707167625427, + 0.6808802485466003 + ], + [ + 0.15689301490783691, + 0.6997910141944885 + ], + [ + 0.16223272681236267, + 0.7187188267707825 + ], + [ + 0.16759774088859558, + 0.7376715540885925 + ], + [ + 0.17303690314292908, + 0.7566697001457214 + ], + [ + 0.17862369120121002, + 0.7757418751716614 + ], + [ + 0.1844315379858017, + 0.7949170470237732 + ], + [ + 0.19053390622138977, + 0.8142240643501282 + ], + [ + 0.19700415432453156, + 0.8336920142173767 + ], + [ + 0.20391567051410675, + 0.8533498644828796 + ], + [ + 0.21134179830551147, + 0.8732268214225769 + ], + [ + 0.21935580670833588, + 0.8933521509170532 + ], + [ + 0.22803090512752533, + 0.9137551784515381 + ], + [ + 0.23744019865989685, + 0.9344654083251953 + ], + [ + 0.24765664339065552, + 0.9555124640464783 + ], + [ + 0.2587530016899109, + 0.9769262671470642 + ], + [ + 0.2708017826080322, + 0.9987370371818542 + ], + [ + 0.2838752567768097, + 1.0209753513336182 + ], + [ + 0.29804527759552, + 1.0436722040176392 + ], + [ + 0.3133833110332489, + 1.0668591260910034 + ], + [ + 0.32996034622192383, + 1.0905684232711792 + ], + [ + 0.3478468656539917, + 1.1148326396942139 + ], + [ + 0.36711281538009644, + 1.1396855115890503 + ], + [ + 0.3878275454044342, + 1.1651611328125 + ], + [ + 0.41000601649284363, + 1.1912426948547363 + ], + [ + 0.4336090087890625, + 1.2178623676300049 + ], + [ + 0.45859649777412415, + 1.24495267868042 + ], + [ + 0.4849279224872589, + 1.272446632385254 + ], + [ + 0.5125620365142822, + 1.300277590751648 + ], + [ + 0.5414570569992065, + 1.3283785581588745 + ], + [ + 0.571570634841919, + 1.3566826581954956 + ], + [ + 0.6028600931167603, + 1.3851218223571777 + ], + [ + 0.6352822780609131, + 1.4136276245117188 + ], + [ + 0.6687941551208496, + 1.4421300888061523 + ], + [ + 0.7033523917198181, + 1.4705579280853271 + ], + [ + 0.7389140725135803, + 1.4988375902175903 + ], + [ + 0.7754364013671875, + 1.5268937349319458 + ], + [ + 0.8128772974014282, + 1.554648518562317 + ], + [ + 0.8511953353881836, + 1.5820214748382568 + ], + [ + 0.8903500437736511, + 1.6089295148849487 + ], + [ + 0.930302083492279, + 1.6352869272232056 + ], + [ + 0.9710133075714111, + 1.6610050201416016 + ], + [ + 1.0124471187591553, + 1.685992956161499 + ], + [ + 1.0545682907104492, + 1.7101572751998901 + ], + [ + 1.0973432064056396, + 1.7334026098251343 + ], + [ + 1.1407400369644165, + 1.755631923675537 + ], + [ + 1.1847283840179443, + 1.7767466306686401 + ], + [ + 1.229279637336731, + 1.7966474294662476 + ], + [ + 1.2743664979934692, + 1.8152343034744263 + ], + [ + 1.3199630975723267, + 1.8324071168899536 + ], + [ + 1.3660447597503662, + 1.848065733909607 + ], + [ + 1.4125876426696777, + 1.8621103763580322 + ], + [ + 1.4595685005187988, + 1.8744415044784546 + ], + [ + 1.5069485902786255, + 1.8850340843200684 + ], + [ + 1.5546728372573853, + 1.8939374685287476 + ], + [ + 1.6026861667633057, + 1.9012004137039185 + ], + [ + 1.650932788848877, + 1.9068700075149536 + ], + [ + 1.6993565559387207, + 1.9109913110733032 + ], + [ + 1.7479000091552734, + 1.9136062860488892 + ], + [ + 1.7965046167373657, + 1.9147534370422363 + ], + [ + 1.8451100587844849, + 1.9144660234451294 + ], + [ + 1.893654465675354, + 1.9127724170684814 + ], + [ + 1.942073941230774, + 1.9096946716308594 + ], + [ + 1.990302324295044, + 1.9052486419677734 + ], + [ + 2.03827166557312, + 1.8994430303573608 + ], + [ + 2.0859110355377197, + 1.8922799825668335 + ], + [ + 2.133148193359375, + 1.8837546110153198 + ], + [ + 2.179908037185669, + 1.8738549947738647 + ], + [ + 2.2261133193969727, + 1.862563133239746 + ], + [ + 2.27168607711792, + 1.8498553037643433 + ], + [ + 2.316545248031616, + 1.8357020616531372 + ], + [ + 2.360609769821167, + 1.8200697898864746 + ], + [ + 2.403796911239624, + 1.8029208183288574 + ], + [ + 2.44602370262146, + 1.7842146158218384 + ], + [ + 2.4872069358825684, + 1.7639081478118896 + ], + [ + 2.5272626876831055, + 1.7419570684432983 + ], + [ + 2.566108465194702, + 1.7183157205581665 + ], + [ + 2.603734254837036, + 1.693010687828064 + ], + [ + 2.640204906463623, + 1.6661417484283447 + ], + [ + 2.6755847930908203, + 1.6378077268600464 + ], + [ + 2.7099392414093018, + 1.6081076860427856 + ], + [ + 2.743333101272583, + 1.5771397352218628 + ], + [ + 2.7758309841156006, + 1.5450016260147095 + ], + [ + 2.80749773979187, + 1.5117899179458618 + ], + [ + 2.8383967876434326, + 1.477600336074829 + ], + [ + 2.868591785430908, + 1.4425268173217773 + ], + [ + 2.8981447219848633, + 1.4066622257232666 + ], + [ + 2.9271178245544434, + 1.3700973987579346 + ], + [ + 2.9555718898773193, + 1.3329222202301025 + ], + [ + 2.983566999435425, + 1.2952247858047485 + ], + [ + 3.011162757873535, + 1.2570923566818237 + ], + [ + 3.038418769836426, + 1.2186110019683838 + ], + [ + 3.0653929710388184, + 1.1798664331436157 + ], + [ + 3.092144250869751, + 1.1409443616867065 + ], + [ + 3.118730306625366, + 1.1019304990768433 + ], + [ + 3.1451311111450195, + 1.062860131263733 + ], + [ + 3.171248197555542, + 1.0237183570861816 + ], + [ + 3.1969823837280273, + 0.9844915866851807 + ], + [ + 3.222233295440674, + 0.945167064666748 + ], + [ + 3.246898889541626, + 0.905733585357666 + ], + [ + 3.270875930786133, + 0.8661811947822571 + ], + [ + 3.294057846069336, + 0.826501190662384 + ], + [ + 3.3163373470306396, + 0.7866860032081604 + ], + [ + 3.3376033306121826, + 0.7467291951179504 + ], + [ + 3.357743263244629, + 0.7066251039505005 + ], + [ + 3.3766419887542725, + 0.6663689613342285 + ], + [ + 3.394181966781616, + 0.6259563565254211 + ], + [ + 3.4102442264556885, + 0.5853835344314575 + ], + [ + 3.424708127975464, + 0.5446467995643616 + ], + [ + 3.4374516010284424, + 0.5037427544593811 + ], + [ + 3.448352098464966, + 0.46266797184944153 + ], + [ + 3.457287073135376, + 0.42141908407211304 + ], + [ + 3.4641330242156982, + 0.3799927234649658 + ], + [ + 3.468876838684082, + 0.33839157223701477 + ], + [ + 3.471616506576538, + 0.2966245114803314 + ], + [ + 3.4724483489990234, + 0.2547004222869873 + ], + [ + 3.4714694023132324, + 0.21262840926647186 + ], + [ + 3.4687745571136475, + 0.17041781544685364 + ], + [ + 3.4644577503204346, + 0.1280783712863922 + ], + [ + 3.4586100578308105, + 0.0856202244758606 + ], + [ + 3.4513206481933594, + 0.043054141104221344 + ], + [ + 3.442674398422241, + 0.0003915314737241715 + ], + [ + 3.432753562927246, + -0.04235544055700302 + ], + [ + 3.421635389328003, + -0.08517380803823471 + ], + [ + 3.409393072128296, + -0.12804976105690002 + ], + [ + 3.3960955142974854, + -0.17096871137619019 + ], + [ + 3.3818066120147705, + -0.21391519904136658 + ], + [ + 3.366586685180664, + -0.25687310099601746 + ], + [ + 3.3504908084869385, + -0.29982560873031616 + ], + [ + 3.333570718765259, + -0.34275543689727783 + ], + [ + 3.315875291824341, + -0.3856448531150818 + ], + [ + 3.297449827194214, + -0.42847591638565063 + ], + [ + 3.278337240219116, + -0.47123050689697266 + ], + [ + 3.2585792541503906, + -0.5138905048370361 + ], + [ + 3.238215923309326, + -0.5564379692077637 + ], + [ + 3.217292308807373, + -0.5988707542419434 + ], + [ + 3.1958582401275635, + -0.6412028074264526 + ], + [ + 3.1739635467529297, + -0.6834480166435242 + ], + [ + 3.1516590118408203, + -0.7256200909614563 + ], + [ + 3.1289961338043213, + -0.7677323818206787 + ], + [ + 3.1060280799865723, + -0.8097975850105286 + ], + [ + 3.082807779312134, + -0.8518276214599609 + ], + [ + 3.0593905448913574, + -0.8938331604003906 + ], + [ + 3.0358314514160156, + -0.9358235001564026 + ], + [ + 3.0062689781188965, + -0.9883013367652893 + ], + [ + 2.9885144233703613, + -1.0197867155075073 + ] + ] + } +] diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/05_root_path/meta.json b/kimodo/assets/demo/examples/kimodo-g1-rp/05_root_path/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..780f4c47efcaa9b5f442d677e4eae4d415a02243 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/05_root_path/meta.json @@ -0,0 +1,12 @@ +{ + "text": "Initially standing still and calm, the person then starts jogging in a counterclockwise arc.", + "duration": 6.033333333333333, + "num_samples": 1, + "seed": 62, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/05_root_path/motion.npz b/kimodo/assets/demo/examples/kimodo-g1-rp/05_root_path/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..0cea596e8d90ad2187ea90876e13ac553897ff4c --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/05_root_path/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0eaf2efd5d4120048ef56045c961af932ceb4f72f50034827182f310162164b +size 296914 diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/06_root_waypoints/constraints.json b/kimodo/assets/demo/examples/kimodo-g1-rp/06_root_waypoints/constraints.json new file mode 100644 index 0000000000000000000000000000000000000000..980788ba2768e0854fcc53c620b1a1dbef8ac9d4 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/06_root_waypoints/constraints.json @@ -0,0 +1,29 @@ +[ + { + "type": "root2d", + "frame_indices": [ + 0, + 87, + 169, + 240 + ], + "smooth_root_2d": [ + [ + 0.037946805357933044, + -0.036908961832523346 + ], + [ + 2.2506563663482666, + 0.06945009529590607 + ], + [ + 2.23332142829895, + -2.0749685764312744 + ], + [ + 4.0815324783325195, + -2.273184061050415 + ] + ] + } +] diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/06_root_waypoints/meta.json b/kimodo/assets/demo/examples/kimodo-g1-rp/06_root_waypoints/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..dcf09b1b72c4483ef6d48b605de90978d75e34c6 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/06_root_waypoints/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person is walking while carrying a small object in their left hand", + "duration": 8.033333333333333, + "num_samples": 1, + "seed": 61, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/06_root_waypoints/motion.npz b/kimodo/assets/demo/examples/kimodo-g1-rp/06_root_waypoints/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..c36e56c21ecf0db2d282d77594c3730721e38bc4 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/06_root_waypoints/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eda98d489984ac45289336207dafe3215757e7bc75b770a6a96239e093b4a0a9 +size 395074 diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/07_text_terrain/meta.json b/kimodo/assets/demo/examples/kimodo-g1-rp/07_text_terrain/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..77d404bad05ebeaab289329297837b9061c95d38 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/07_text_terrain/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person begins walking up the stairs", + "duration": 3.5, + "num_samples": 1, + "seed": 44, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/07_text_terrain/motion.npz b/kimodo/assets/demo/examples/kimodo-g1-rp/07_text_terrain/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..90d40abf09c5d15556ff846099019b9306c9bb35 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/07_text_terrain/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1c2920338a8ecaa0f8c25a24f06bdca88db8b406c33a68930a459d64f6beda8 +size 172578 diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/08_text_object/meta.json b/kimodo/assets/demo/examples/kimodo-g1-rp/08_text_object/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..e0d64e0bac7eeb3c772633e3b4d68190bdcb83a9 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/08_text_object/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person picks up an object from low on their left side and places it up high", + "duration": 5.033333333333333, + "num_samples": 1, + "seed": 47, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-g1-rp/08_text_object/motion.npz b/kimodo/assets/demo/examples/kimodo-g1-rp/08_text_object/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..8f6ff1bdf4aa2ca3dce29828f423bf94014a4d73 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-g1-rp/08_text_object/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23032acb7b0db2c194c5d967529ac76ac0dc63920903fb504375090b9b83aba1 +size 247834 diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/01_single_text_prompt/meta.json b/kimodo/assets/demo/examples/kimodo-soma-rp/01_single_text_prompt/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..081a217a0f0c7fe95b905e939905b61a853c746b --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/01_single_text_prompt/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person runs forward and then leaps over an obstacle in front of them.", + "duration": 5.0, + "num_samples": 1, + "seed": 42, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/01_single_text_prompt/motion.npz b/kimodo/assets/demo/examples/kimodo-soma-rp/01_single_text_prompt/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..e279853246758f010a2c9fb99aa2ae8d0c1717a5 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/01_single_text_prompt/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37b6f13ed9c9f55696a9f9c896e8770fbdbd622864b08cd075e4eaf3a6a21f95 +size 217398 diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/02_multi_text_prompt/meta.json b/kimodo/assets/demo/examples/kimodo-soma-rp/02_multi_text_prompt/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..9c3ffc55219a89338f74a798ce090c105d2652fa --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/02_multi_text_prompt/meta.json @@ -0,0 +1,18 @@ +{ + "texts": [ + "A person is walking forward casually.", + "A person turns to the right and starts sneakily moving forward" + ], + "durations": [ + 3.533333333333333, + 4.033333333333333 + ], + "num_samples": 1, + "seed": 42, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/02_multi_text_prompt/motion.npz b/kimodo/assets/demo/examples/kimodo-soma-rp/02_multi_text_prompt/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..6672aeee962021c9992791db7023d345d0ba437d --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/02_multi_text_prompt/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c7731d7e84fb0944fb23a71bff2086e766f2f7884bd26e1b1ce36710ff84537 +size 328586 diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/03_full_body_keyframes/constraints.json b/kimodo/assets/demo/examples/kimodo-soma-rp/03_full_body_keyframes/constraints.json new file mode 100644 index 0000000000000000000000000000000000000000..ac94699138e38d3fdac9f468dcdfad448bacbf87 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/03_full_body_keyframes/constraints.json @@ -0,0 +1,337 @@ +[ + { + "type": "fullbody", + "frame_indices": [ + 79, + 134 + ], + "local_joints_rot": [ + [ + [ + 0.2765098512172699, + 0.3728594183921814, + -0.3292054831981659 + ], + [ + 0.35604047775268555, + 0.018222831189632416, + -0.054862238466739655 + ], + [ + 0.12065527588129044, + -0.027457308024168015, + -0.06907646358013153 + ], + [ + 0.6048485636711121, + -0.11472737789154053, + -0.19573566317558289 + ], + [ + -0.12398597598075867, + 0.03840772435069084, + 0.18822282552719116 + ], + [ + -0.06553511321544647, + 0.13032270967960358, + 0.04257704317569733 + ], + [ + -0.24969959259033203, + 0.06990747153759003, + 0.13426002860069275 + ], + [ + -0.002762501360848546, + 0.0010064352536574006, + -0.0012083332985639572 + ], + [ + -0.18770116567611694, + -0.06528781354427338, + 0.006136383395642042 + ], + [ + -0.18933561444282532, + 0.06753389537334442, + -0.00862747710198164 + ], + [ + 0.1765439361333847, + -0.5079103708267212, + 0.11742556095123291 + ], + [ + -0.6833809614181519, + -0.36341744661331177, + -0.09875624626874924 + ], + [ + -0.004083660896867514, + -0.2955799102783203, + 0.007416445296257734 + ], + [ + -0.46948903799057007, + 0.0019703502766788006, + 0.2218078076839447 + ], + [ + 0.15589098632335663, + 0.29247695207595825, + -0.2839103043079376 + ], + [ + -0.006183772347867489, + 0.039787642657756805, + -1.0509610176086426 + ], + [ + 0.28110796213150024, + -0.01673225313425064, + 0.05465283617377281 + ], + [ + 0.4582408368587494, + 0.6058111786842346, + 1.040449619293213 + ], + [ + -0.016165010631084442, + 0.7843144536018372, + 0.007565980777144432 + ], + [ + -0.21160456538200378, + 0.009858175180852413, + 0.022257711738348007 + ], + [ + 0.08559019863605499, + -0.26941442489624023, + 0.28404051065444946 + ], + [ + -0.0722564086318016, + -0.055347055196762085, + 0.8767912983894348 + ], + [ + -0.9036330580711365, + -0.19308030605316162, + 0.6912829875946045 + ], + [ + 1.7018375396728516, + -0.052370231598615646, + 0.0016176343197003007 + ], + [ + -0.6713079810142517, + -0.22423480451107025, + -0.17199599742889404 + ], + [ + -0.2397085577249527, + -0.04111046716570854, + 0.02976534143090248 + ], + [ + -1.4084941148757935, + -0.42399686574935913, + 0.23780424892902374 + ], + [ + 1.488803744316101, + -0.006882219575345516, + 0.005796314682811499 + ], + [ + -0.34890878200531006, + 0.25402817130088806, + -0.10165958851575851 + ], + [ + -0.017090337350964546, + 0.013983047567307949, + -0.02469288557767868 + ] + ], + [ + [ + -0.10219376534223557, + 0.15241079032421112, + -0.1140606626868248 + ], + [ + -0.07097288966178894, + -0.023205779492855072, + 0.014893154613673687 + ], + [ + -0.11436910182237625, + -0.07182353734970093, + -0.024793410673737526 + ], + [ + 0.32571300864219666, + -0.11312247067689896, + -0.017911700531840324 + ], + [ + 0.036515623331069946, + -0.0007576555362902582, + 0.14029929041862488 + ], + [ + -0.06553909182548523, + 0.07225329428911209, + 0.0065536051988601685 + ], + [ + -0.09946814924478531, + 0.02283940091729164, + 0.060293473303318024 + ], + [ + -0.0007363191107288003, + 0.0019088855478912592, + 0.00034123589284718037 + ], + [ + -0.18651022017002106, + -0.06423485279083252, + 0.0069741918705403805 + ], + [ + -0.18586836755275726, + 0.06800899654626846, + -0.0060585117898881435 + ], + [ + 0.23363706469535828, + -0.20687633752822876, + -0.07240967452526093 + ], + [ + -0.3135974407196045, + -0.2623864710330963, + -1.0657873153686523 + ], + [ + -0.012310811318457127, + -1.6650079488754272, + -0.010509567335247993 + ], + [ + -0.8171713352203369, + -0.2551392912864685, + 0.08705981075763702 + ], + [ + 0.13723036646842957, + 0.2864063084125519, + -0.2900709807872772 + ], + [ + -0.005930017679929733, + 0.05293968319892883, + -1.0459250211715698 + ], + [ + 0.24218180775642395, + 0.02018338069319725, + 0.1226770281791687 + ], + [ + 0.3315959572792053, + 0.3782292902469635, + 1.2296319007873535 + ], + [ + -0.0014527677558362484, + 0.3045952022075653, + -0.0014049106976017356 + ], + [ + -0.20010970532894135, + -0.07485076785087585, + 0.0041703470051288605 + ], + [ + 0.08470325917005539, + -0.3079097270965576, + 0.29375413060188293 + ], + [ + -0.09725581854581833, + -0.055068179965019226, + 0.8742175698280334 + ], + [ + 0.4040503203868866, + -0.016711091622710228, + 0.21672509610652924 + ], + [ + 0.5082376599311829, + -0.013459251262247562, + 0.004872385878115892 + ], + [ + 0.1745426058769226, + -0.24501416087150574, + -0.003703102469444275 + ], + [ + -0.33402949571609497, + -0.035541169345378876, + 0.032360970973968506 + ], + [ + -0.37681734561920166, + 0.02067263424396515, + 0.10783999413251877 + ], + [ + 0.4257254898548126, + 0.0016118268249556422, + 0.0033562832977622747 + ], + [ + 0.04139057174324989, + 0.032555095851421356, + 0.04008425772190094 + ], + [ + -0.03090120106935501, + 0.01570875011384487, + -0.024774780496954918 + ] + ] + ], + "root_positions": [ + [ + -0.18697306513786316, + 0.7126776576042175, + 1.1559109687805176 + ], + [ + -0.014062155969440937, + 0.9611971974372864, + 2.898127555847168 + ] + ], + "smooth_root_2d": [ + [ + -0.18697306513786316, + 1.1559109687805176 + ], + [ + -0.014062155969440937, + 2.898127555847168 + ] + ] + } +] diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/03_full_body_keyframes/meta.json b/kimodo/assets/demo/examples/kimodo-soma-rp/03_full_body_keyframes/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..0d39f2cf66e3b182a6a7711dbeeab14226abd4df --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/03_full_body_keyframes/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person walks forward and picks something up from the ground", + "duration": 5.0, + "num_samples": 1, + "seed": 43, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/03_full_body_keyframes/motion.npz b/kimodo/assets/demo/examples/kimodo-soma-rp/03_full_body_keyframes/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..865fde488299e602772b9ed65b1900f878941407 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/03_full_body_keyframes/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4c7f89c9fd5394a8b635625815add45acfda0bf568d8ce41f42892034a99cd2 +size 217398 diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/04_ee_constraint/constraints.json b/kimodo/assets/demo/examples/kimodo-soma-rp/04_ee_constraint/constraints.json new file mode 100644 index 0000000000000000000000000000000000000000..ba060ea5f99c2afd53950f8d5db7165e695a9c01 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/04_ee_constraint/constraints.json @@ -0,0 +1,1342 @@ +[ + { + "type": "right-foot", + "frame_indices": [ + 28, + 94 + ], + "local_joints_rot": [ + [ + [ + 0.14788010716438293, + -0.010833931155502796, + -0.01388303842395544 + ], + [ + -0.03901153802871704, + 0.0003969503741245717, + -0.00016447225061710924 + ], + [ + -0.09507032483816147, + 0.008639314211905003, + -0.0073561337776482105 + ], + [ + 0.21237806975841522, + -0.02139095962047577, + -0.01700877584517002 + ], + [ + -0.20991119742393494, + 0.06551700085401535, + -0.05272415280342102 + ], + [ + -0.06337061524391174, + 0.05204080045223236, + 0.014292852953076363 + ], + [ + 0.07047945261001587, + 0.08330309391021729, + -0.002013514516875148 + ], + [ + -0.0019600456580519676, + -0.0013381227618083358, + -2.7628393581835553e-06 + ], + [ + -0.18709787726402283, + -0.06659803539514542, + 0.0078862514346838 + ], + [ + -0.18698126077651978, + 0.06395528465509415, + -0.008215037174522877 + ], + [ + 0.08230585604906082, + -0.38376951217651367, + 0.05542140454053879 + ], + [ + -0.7260366082191467, + -0.24878422915935516, + -0.35609468817710876 + ], + [ + 0.004249485209584236, + -0.4476320147514343, + -0.018469776958227158 + ], + [ + -0.9212101697921753, + -0.1470143049955368, + 0.5044775605201721 + ], + [ + 0.14870156347751617, + 0.2985619604587555, + -0.29298385977745056 + ], + [ + 0.001955621177330613, + 0.055549487471580505, + -1.0630463361740112 + ], + [ + 0.11859050393104553, + 0.46535199880599976, + -0.030845582485198975 + ], + [ + -0.7298654317855835, + 0.5346517562866211, + 0.2791443467140198 + ], + [ + 0.008972911164164543, + 0.48752307891845703, + 0.01847967691719532 + ], + [ + -0.5805565118789673, + -0.08708631247282028, + -0.15088550746440887 + ], + [ + 0.08582834899425507, + -0.2886488735675812, + 0.2854447066783905 + ], + [ + -0.0898093581199646, + -0.05874425172805786, + 0.8657776117324829 + ], + [ + -0.3135877549648285, + 0.07464626431465149, + 0.0517989918589592 + ], + [ + 0.29447537660598755, + -0.003720453940331936, + 0.0011728419922292233 + ], + [ + -0.12890003621578217, + 0.0839272066950798, + -0.090343177318573 + ], + [ + 0.008360159583389759, + -0.03457032889127731, + 0.02827553078532219 + ], + [ + -0.3120643198490143, + -0.01133657619357109, + -0.03218594938516617 + ], + [ + 0.2538771331310272, + 0.0018040596041828394, + 0.0009352069464512169 + ], + [ + -0.0887608677148819, + -0.03465384244918823, + 0.07154331356287003 + ], + [ + 0.01681467890739441, + 0.01778421923518181, + -0.025033073499798775 + ] + ], + [ + [ + 0.21243979036808014, + 1.0922467708587646, + -0.05739659443497658 + ], + [ + -0.04288899898529053, + 0.019888481125235558, + -0.014078406617045403 + ], + [ + -0.09594971686601639, + 0.10335114598274231, + -0.007776615675538778 + ], + [ + 0.2422163188457489, + 0.08445896953344345, + -0.05605608597397804 + ], + [ + -0.14986605942249298, + 0.10279522091150284, + -0.19410337507724762 + ], + [ + -0.07278254628181458, + 0.00021229058620519936, + -0.0064666238613426685 + ], + [ + -0.18101167678833008, + -0.047196485102176666, + 0.09371022135019302 + ], + [ + -0.0013136633206158876, + -0.0020103836432099342, + -0.0002618256548885256 + ], + [ + -0.1867513209581375, + -0.0681525468826294, + 0.0023792991414666176 + ], + [ + -0.18714284896850586, + 0.06443598866462708, + -0.003183535533025861 + ], + [ + 0.1040755957365036, + -0.1164601668715477, + -0.08953910320997238 + ], + [ + -0.7818892598152161, + -0.40082883834838867, + -0.40901198983192444 + ], + [ + 0.0014971806667745113, + -0.7006690502166748, + -0.003588718129321933 + ], + [ + -0.7653300762176514, + -0.030549153685569763, + 0.5779297947883606 + ], + [ + 0.1444747895002365, + 0.30648332834243774, + -0.2944350242614746 + ], + [ + 0.00627485616132617, + 0.05844533443450928, + -1.0504485368728638 + ], + [ + 0.16790169477462769, + 0.6803913116455078, + -0.0802350640296936 + ], + [ + -0.7650246620178223, + 0.2571314871311188, + 0.044474273920059204 + ], + [ + 0.00177879654802382, + 0.32478848099708557, + 0.024663111194968224 + ], + [ + -1.1130585670471191, + 0.06198093295097351, + -0.1499929279088974 + ], + [ + 0.09419120848178864, + -0.28672322630882263, + 0.2861841320991516 + ], + [ + -0.08110660314559937, + -0.06315471976995468, + 0.8641197085380554 + ], + [ + -0.4702282249927521, + -0.2976788580417633, + -0.08966172486543655 + ], + [ + 0.2188275307416916, + -0.010813144035637379, + -0.0024994502309709787 + ], + [ + 0.12644176185131073, + -0.4933742582798004, + -0.23269610106945038 + ], + [ + -0.05216464772820473, + -0.03182952478528023, + 0.026469329372048378 + ], + [ + -0.21055173873901367, + -0.5854666233062744, + -0.08316371589899063 + ], + [ + 0.2703852653503418, + -0.0070351893082261086, + 0.00034556735772639513 + ], + [ + -0.20080512762069702, + -0.5529999136924744, + 0.08794122189283371 + ], + [ + -0.020619722083210945, + 0.01961597241461277, + -0.02498687617480755 + ] + ] + ], + "root_positions": [ + [ + 0.006224155426025391, + 1.0099574327468872, + 0.0004121592501178384 + ], + [ + 0.025673866271972656, + 1.0039517879486084, + 0.0002174415858462453 + ] + ], + "smooth_root_2d": [ + [ + 0.006224155426025391, + 0.0004121592501178384 + ], + [ + 0.025673866271972656, + 0.0002174415858462453 + ] + ] + }, + { + "type": "left-foot", + "frame_indices": [ + 28, + 94 + ], + "local_joints_rot": [ + [ + [ + 0.14788010716438293, + -0.010833931155502796, + -0.01388303842395544 + ], + [ + -0.03901153802871704, + 0.0003969503741245717, + -0.00016447225061710924 + ], + [ + -0.09507032483816147, + 0.008639314211905003, + -0.0073561337776482105 + ], + [ + 0.21237806975841522, + -0.02139095962047577, + -0.01700877584517002 + ], + [ + -0.20991119742393494, + 0.06551700085401535, + -0.05272415280342102 + ], + [ + -0.06337061524391174, + 0.05204080045223236, + 0.014292852953076363 + ], + [ + 0.07047945261001587, + 0.08330309391021729, + -0.002013514516875148 + ], + [ + -0.0019600456580519676, + -0.0013381227618083358, + -2.7628393581835553e-06 + ], + [ + -0.18709787726402283, + -0.06659803539514542, + 0.0078862514346838 + ], + [ + -0.18698126077651978, + 0.06395528465509415, + -0.008215037174522877 + ], + [ + 0.08230585604906082, + -0.38376951217651367, + 0.05542140454053879 + ], + [ + -0.7260366082191467, + -0.24878422915935516, + -0.35609468817710876 + ], + [ + 0.004249485209584236, + -0.4476320147514343, + -0.018469776958227158 + ], + [ + -0.9212101697921753, + -0.1470143049955368, + 0.5044775605201721 + ], + [ + 0.14870156347751617, + 0.2985619604587555, + -0.29298385977745056 + ], + [ + 0.001955621177330613, + 0.055549487471580505, + -1.0630463361740112 + ], + [ + 0.11859050393104553, + 0.46535199880599976, + -0.030845582485198975 + ], + [ + -0.7298654317855835, + 0.5346517562866211, + 0.2791443467140198 + ], + [ + 0.008972911164164543, + 0.48752307891845703, + 0.01847967691719532 + ], + [ + -0.5805565118789673, + -0.08708631247282028, + -0.15088550746440887 + ], + [ + 0.08582834899425507, + -0.2886488735675812, + 0.2854447066783905 + ], + [ + -0.0898093581199646, + -0.05874425172805786, + 0.8657776117324829 + ], + [ + -0.3135877549648285, + 0.07464626431465149, + 0.0517989918589592 + ], + [ + 0.29447537660598755, + -0.003720453940331936, + 0.0011728419922292233 + ], + [ + -0.12890003621578217, + 0.0839272066950798, + -0.090343177318573 + ], + [ + 0.008360159583389759, + -0.03457032889127731, + 0.02827553078532219 + ], + [ + -0.3120643198490143, + -0.01133657619357109, + -0.03218594938516617 + ], + [ + 0.2538771331310272, + 0.0018040596041828394, + 0.0009352069464512169 + ], + [ + -0.0887608677148819, + -0.03465384244918823, + 0.07154331356287003 + ], + [ + 0.01681467890739441, + 0.01778421923518181, + -0.025033073499798775 + ] + ], + [ + [ + 0.21243979036808014, + 1.0922467708587646, + -0.05739659443497658 + ], + [ + -0.04288899898529053, + 0.019888481125235558, + -0.014078406617045403 + ], + [ + -0.09594971686601639, + 0.10335114598274231, + -0.007776615675538778 + ], + [ + 0.2422163188457489, + 0.08445896953344345, + -0.05605608597397804 + ], + [ + -0.14986605942249298, + 0.10279522091150284, + -0.19410337507724762 + ], + [ + -0.07278254628181458, + 0.00021229058620519936, + -0.0064666238613426685 + ], + [ + -0.18101167678833008, + -0.047196485102176666, + 0.09371022135019302 + ], + [ + -0.0013136633206158876, + -0.0020103836432099342, + -0.0002618256548885256 + ], + [ + -0.1867513209581375, + -0.0681525468826294, + 0.0023792991414666176 + ], + [ + -0.18714284896850586, + 0.06443598866462708, + -0.003183535533025861 + ], + [ + 0.1040755957365036, + -0.1164601668715477, + -0.08953910320997238 + ], + [ + -0.7818892598152161, + -0.40082883834838867, + -0.40901198983192444 + ], + [ + 0.0014971806667745113, + -0.7006690502166748, + -0.003588718129321933 + ], + [ + -0.7653300762176514, + -0.030549153685569763, + 0.5779297947883606 + ], + [ + 0.1444747895002365, + 0.30648332834243774, + -0.2944350242614746 + ], + [ + 0.00627485616132617, + 0.05844533443450928, + -1.0504485368728638 + ], + [ + 0.16790169477462769, + 0.6803913116455078, + -0.0802350640296936 + ], + [ + -0.7650246620178223, + 0.2571314871311188, + 0.044474273920059204 + ], + [ + 0.00177879654802382, + 0.32478848099708557, + 0.024663111194968224 + ], + [ + -1.1130585670471191, + 0.06198093295097351, + -0.1499929279088974 + ], + [ + 0.09419120848178864, + -0.28672322630882263, + 0.2861841320991516 + ], + [ + -0.08110660314559937, + -0.06315471976995468, + 0.8641197085380554 + ], + [ + -0.4702282249927521, + -0.2976788580417633, + -0.08966172486543655 + ], + [ + 0.2188275307416916, + -0.010813144035637379, + -0.0024994502309709787 + ], + [ + 0.12644176185131073, + -0.4933742582798004, + -0.23269610106945038 + ], + [ + -0.05216464772820473, + -0.03182952478528023, + 0.026469329372048378 + ], + [ + -0.21055173873901367, + -0.5854666233062744, + -0.08316371589899063 + ], + [ + 0.2703852653503418, + -0.0070351893082261086, + 0.00034556735772639513 + ], + [ + -0.20080512762069702, + -0.5529999136924744, + 0.08794122189283371 + ], + [ + -0.020619722083210945, + 0.01961597241461277, + -0.02498687617480755 + ] + ] + ], + "root_positions": [ + [ + 0.006224155426025391, + 1.0099574327468872, + 0.0004121592501178384 + ], + [ + 0.025673866271972656, + 1.0039517879486084, + 0.0002174415858462453 + ] + ], + "smooth_root_2d": [ + [ + 0.006224155426025391, + 0.0004121592501178384 + ], + [ + 0.025673866271972656, + 0.0002174415858462453 + ] + ] + }, + { + "type": "left-hand", + "frame_indices": [ + 28, + 94 + ], + "local_joints_rot": [ + [ + [ + 0.14788010716438293, + -0.010833931155502796, + -0.01388303842395544 + ], + [ + -0.03901153802871704, + 0.0003969503741245717, + -0.00016447225061710924 + ], + [ + -0.09507032483816147, + 0.008639314211905003, + -0.0073561337776482105 + ], + [ + 0.21237806975841522, + -0.02139095962047577, + -0.01700877584517002 + ], + [ + -0.20991119742393494, + 0.06551700085401535, + -0.05272415280342102 + ], + [ + -0.06337061524391174, + 0.05204080045223236, + 0.014292852953076363 + ], + [ + 0.07047945261001587, + 0.08330309391021729, + -0.002013514516875148 + ], + [ + -0.0019600456580519676, + -0.0013381227618083358, + -2.7628393581835553e-06 + ], + [ + -0.18709787726402283, + -0.06659803539514542, + 0.0078862514346838 + ], + [ + -0.18698126077651978, + 0.06395528465509415, + -0.008215037174522877 + ], + [ + 0.08230585604906082, + -0.38376951217651367, + 0.05542140454053879 + ], + [ + -0.7260366082191467, + -0.24878422915935516, + -0.35609468817710876 + ], + [ + 0.004249485209584236, + -0.4476320147514343, + -0.018469776958227158 + ], + [ + -0.9212101697921753, + -0.1470143049955368, + 0.5044775605201721 + ], + [ + 0.14870156347751617, + 0.2985619604587555, + -0.29298385977745056 + ], + [ + 0.001955621177330613, + 0.055549487471580505, + -1.0630463361740112 + ], + [ + 0.11859050393104553, + 0.46535199880599976, + -0.030845582485198975 + ], + [ + -0.7298654317855835, + 0.5346517562866211, + 0.2791443467140198 + ], + [ + 0.008972911164164543, + 0.48752307891845703, + 0.01847967691719532 + ], + [ + -0.5805565118789673, + -0.08708631247282028, + -0.15088550746440887 + ], + [ + 0.08582834899425507, + -0.2886488735675812, + 0.2854447066783905 + ], + [ + -0.0898093581199646, + -0.05874425172805786, + 0.8657776117324829 + ], + [ + -0.3135877549648285, + 0.07464626431465149, + 0.0517989918589592 + ], + [ + 0.29447537660598755, + -0.003720453940331936, + 0.0011728419922292233 + ], + [ + -0.12890003621578217, + 0.0839272066950798, + -0.090343177318573 + ], + [ + 0.008360159583389759, + -0.03457032889127731, + 0.02827553078532219 + ], + [ + -0.3120643198490143, + -0.01133657619357109, + -0.03218594938516617 + ], + [ + 0.2538771331310272, + 0.0018040596041828394, + 0.0009352069464512169 + ], + [ + -0.0887608677148819, + -0.03465384244918823, + 0.07154331356287003 + ], + [ + 0.01681467890739441, + 0.01778421923518181, + -0.025033073499798775 + ] + ], + [ + [ + 0.21243979036808014, + 1.0922467708587646, + -0.05739659443497658 + ], + [ + -0.04288899898529053, + 0.019888481125235558, + -0.014078406617045403 + ], + [ + -0.09594971686601639, + 0.10335114598274231, + -0.007776615675538778 + ], + [ + 0.2422163188457489, + 0.08445896953344345, + -0.05605608597397804 + ], + [ + -0.14986605942249298, + 0.10279522091150284, + -0.19410337507724762 + ], + [ + -0.07278254628181458, + 0.00021229058620519936, + -0.0064666238613426685 + ], + [ + -0.18101167678833008, + -0.047196485102176666, + 0.09371022135019302 + ], + [ + -0.0013136633206158876, + -0.0020103836432099342, + -0.0002618256548885256 + ], + [ + -0.1867513209581375, + -0.0681525468826294, + 0.0023792991414666176 + ], + [ + -0.18714284896850586, + 0.06443598866462708, + -0.003183535533025861 + ], + [ + 0.1040755957365036, + -0.1164601668715477, + -0.08953910320997238 + ], + [ + -0.7818892598152161, + -0.40082883834838867, + -0.40901198983192444 + ], + [ + 0.0014971806667745113, + -0.7006690502166748, + -0.003588718129321933 + ], + [ + -0.7653300762176514, + -0.030549153685569763, + 0.5779297947883606 + ], + [ + 0.1444747895002365, + 0.30648332834243774, + -0.2944350242614746 + ], + [ + 0.00627485616132617, + 0.05844533443450928, + -1.0504485368728638 + ], + [ + 0.16790169477462769, + 0.6803913116455078, + -0.0802350640296936 + ], + [ + -0.7650246620178223, + 0.2571314871311188, + 0.044474273920059204 + ], + [ + 0.00177879654802382, + 0.32478848099708557, + 0.024663111194968224 + ], + [ + -1.1130585670471191, + 0.06198093295097351, + -0.1499929279088974 + ], + [ + 0.09419120848178864, + -0.28672322630882263, + 0.2861841320991516 + ], + [ + -0.08110660314559937, + -0.06315471976995468, + 0.8641197085380554 + ], + [ + -0.4702282249927521, + -0.2976788580417633, + -0.08966172486543655 + ], + [ + 0.2188275307416916, + -0.010813144035637379, + -0.0024994502309709787 + ], + [ + 0.12644176185131073, + -0.4933742582798004, + -0.23269610106945038 + ], + [ + -0.05216464772820473, + -0.03182952478528023, + 0.026469329372048378 + ], + [ + -0.21055173873901367, + -0.5854666233062744, + -0.08316371589899063 + ], + [ + 0.2703852653503418, + -0.0070351893082261086, + 0.00034556735772639513 + ], + [ + -0.20080512762069702, + -0.5529999136924744, + 0.08794122189283371 + ], + [ + -0.020619722083210945, + 0.01961597241461277, + -0.02498687617480755 + ] + ] + ], + "root_positions": [ + [ + 0.006224155426025391, + 1.0099574327468872, + 0.0004121592501178384 + ], + [ + 0.025673866271972656, + 1.0039517879486084, + 0.0002174415858462453 + ] + ], + "smooth_root_2d": [ + [ + 0.006224155426025391, + 0.0004121592501178384 + ], + [ + 0.025673866271972656, + 0.0002174415858462453 + ] + ] + }, + { + "type": "right-hand", + "frame_indices": [ + 28, + 94 + ], + "local_joints_rot": [ + [ + [ + 0.14788010716438293, + -0.010833931155502796, + -0.01388303842395544 + ], + [ + -0.03901153802871704, + 0.0003969503741245717, + -0.00016447225061710924 + ], + [ + -0.09507032483816147, + 0.008639314211905003, + -0.0073561337776482105 + ], + [ + 0.21237806975841522, + -0.02139095962047577, + -0.01700877584517002 + ], + [ + -0.20991119742393494, + 0.06551700085401535, + -0.05272415280342102 + ], + [ + -0.06337061524391174, + 0.05204080045223236, + 0.014292852953076363 + ], + [ + 0.07047945261001587, + 0.08330309391021729, + -0.002013514516875148 + ], + [ + -0.0019600456580519676, + -0.0013381227618083358, + -2.7628393581835553e-06 + ], + [ + -0.18709787726402283, + -0.06659803539514542, + 0.0078862514346838 + ], + [ + -0.18698126077651978, + 0.06395528465509415, + -0.008215037174522877 + ], + [ + 0.08230585604906082, + -0.38376951217651367, + 0.05542140454053879 + ], + [ + -0.7260366082191467, + -0.24878422915935516, + -0.35609468817710876 + ], + [ + 0.004249485209584236, + -0.4476320147514343, + -0.018469776958227158 + ], + [ + -0.9212101697921753, + -0.1470143049955368, + 0.5044775605201721 + ], + [ + 0.14870156347751617, + 0.2985619604587555, + -0.29298385977745056 + ], + [ + 0.001955621177330613, + 0.055549487471580505, + -1.0630463361740112 + ], + [ + 0.11859050393104553, + 0.46535199880599976, + -0.030845582485198975 + ], + [ + -0.7298654317855835, + 0.5346517562866211, + 0.2791443467140198 + ], + [ + 0.008972911164164543, + 0.48752307891845703, + 0.01847967691719532 + ], + [ + -0.5805565118789673, + -0.08708631247282028, + -0.15088550746440887 + ], + [ + 0.08582834899425507, + -0.2886488735675812, + 0.2854447066783905 + ], + [ + -0.0898093581199646, + -0.05874425172805786, + 0.8657776117324829 + ], + [ + -0.3135877549648285, + 0.07464626431465149, + 0.0517989918589592 + ], + [ + 0.29447537660598755, + -0.003720453940331936, + 0.0011728419922292233 + ], + [ + -0.12890003621578217, + 0.0839272066950798, + -0.090343177318573 + ], + [ + 0.008360159583389759, + -0.03457032889127731, + 0.02827553078532219 + ], + [ + -0.3120643198490143, + -0.01133657619357109, + -0.03218594938516617 + ], + [ + 0.2538771331310272, + 0.0018040596041828394, + 0.0009352069464512169 + ], + [ + -0.0887608677148819, + -0.03465384244918823, + 0.07154331356287003 + ], + [ + 0.01681467890739441, + 0.01778421923518181, + -0.025033073499798775 + ] + ], + [ + [ + 0.21243979036808014, + 1.0922467708587646, + -0.05739659443497658 + ], + [ + -0.04288899898529053, + 0.019888481125235558, + -0.014078406617045403 + ], + [ + -0.09594971686601639, + 0.10335114598274231, + -0.007776615675538778 + ], + [ + 0.2422163188457489, + 0.08445896953344345, + -0.05605608597397804 + ], + [ + -0.14986605942249298, + 0.10279522091150284, + -0.19410337507724762 + ], + [ + -0.07278254628181458, + 0.00021229058620519936, + -0.0064666238613426685 + ], + [ + -0.18101167678833008, + -0.047196485102176666, + 0.09371022135019302 + ], + [ + -0.0013136633206158876, + -0.0020103836432099342, + -0.0002618256548885256 + ], + [ + -0.1867513209581375, + -0.0681525468826294, + 0.0023792991414666176 + ], + [ + -0.18714284896850586, + 0.06443598866462708, + -0.003183535533025861 + ], + [ + 0.1040755957365036, + -0.1164601668715477, + -0.08953910320997238 + ], + [ + -0.7818892598152161, + -0.40082883834838867, + -0.40901198983192444 + ], + [ + 0.0014971806667745113, + -0.7006690502166748, + -0.003588718129321933 + ], + [ + -0.7653300762176514, + -0.030549153685569763, + 0.5779297947883606 + ], + [ + 0.1444747895002365, + 0.30648332834243774, + -0.2944350242614746 + ], + [ + 0.00627485616132617, + 0.05844533443450928, + -1.0504485368728638 + ], + [ + 0.16790169477462769, + 0.6803913116455078, + -0.0802350640296936 + ], + [ + -0.7650246620178223, + 0.2571314871311188, + 0.044474273920059204 + ], + [ + 0.00177879654802382, + 0.32478848099708557, + 0.024663111194968224 + ], + [ + -1.1130585670471191, + 0.06198093295097351, + -0.1499929279088974 + ], + [ + 0.09419120848178864, + -0.28672322630882263, + 0.2861841320991516 + ], + [ + -0.08110660314559937, + -0.06315471976995468, + 0.8641197085380554 + ], + [ + -0.4702282249927521, + -0.2976788580417633, + -0.08966172486543655 + ], + [ + 0.2188275307416916, + -0.010813144035637379, + -0.0024994502309709787 + ], + [ + 0.12644176185131073, + -0.4933742582798004, + -0.23269610106945038 + ], + [ + -0.05216464772820473, + -0.03182952478528023, + 0.026469329372048378 + ], + [ + -0.21055173873901367, + -0.5854666233062744, + -0.08316371589899063 + ], + [ + 0.2703852653503418, + -0.0070351893082261086, + 0.00034556735772639513 + ], + [ + -0.20080512762069702, + -0.5529999136924744, + 0.08794122189283371 + ], + [ + -0.020619722083210945, + 0.01961597241461277, + -0.02498687617480755 + ] + ] + ], + "root_positions": [ + [ + 0.006224155426025391, + 1.0099574327468872, + 0.0004121592501178384 + ], + [ + 0.025673866271972656, + 1.0039517879486084, + 0.0002174415858462453 + ] + ], + "smooth_root_2d": [ + [ + 0.006224155426025391, + 0.0004121592501178384 + ], + [ + 0.025673866271972656, + 0.0002174415858462453 + ] + ] + } +] diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/04_ee_constraint/meta.json b/kimodo/assets/demo/examples/kimodo-soma-rp/04_ee_constraint/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..47fe3c2c9820e527adce1c1a3c0e25f3ea148c9b --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/04_ee_constraint/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person picks up an object in front of them with two hands and places it to the left side", + "duration": 5.033333333333333, + "num_samples": 1, + "seed": 48, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/04_ee_constraint/motion.npz b/kimodo/assets/demo/examples/kimodo-soma-rp/04_ee_constraint/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..8804910b767757d093a88458b12c0f5b9cda5cc4 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/04_ee_constraint/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40633fe04ac88b96ce8ce202d95870fa4012135721d32777cbb3e198e590c114 +size 218842 diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/05_root_path/constraints.json b/kimodo/assets/demo/examples/kimodo-soma-rp/05_root_path/constraints.json new file mode 100644 index 0000000000000000000000000000000000000000..940318b5d8fd4b11ce4434eab428c529c35e18d1 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/05_root_path/constraints.json @@ -0,0 +1,1509 @@ +[ + { + "type": "root2d", + "frame_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299 + ], + "smooth_root_2d": [ + [ + 0.0720488652586937, + 0.005473949480801821 + ], + [ + 0.08682604879140854, + 0.03799768537282944 + ], + [ + 0.10160323977470398, + 0.07052142173051834 + ], + [ + 0.11638043075799942, + 0.10304517298936844 + ], + [ + 0.13115762174129486, + 0.13556894659996033 + ], + [ + 0.1459348350763321, + 0.1680927276611328 + ], + [ + 0.16071203351020813, + 0.20061656832695007 + ], + [ + 0.17548926174640656, + 0.23314043879508972 + ], + [ + 0.19026650488376617, + 0.26566436886787415 + ], + [ + 0.2050437480211258, + 0.29818838834762573 + ], + [ + 0.2198210209608078, + 0.3307124972343445 + ], + [ + 0.234598308801651, + 0.3632366955280304 + ], + [ + 0.2493756115436554, + 0.39576101303100586 + ], + [ + 0.2641529440879822, + 0.42828547954559326 + ], + [ + 0.27893027663230896, + 0.4608100950717926 + ], + [ + 0.29370763897895813, + 0.4933348596096039 + ], + [ + 0.3084850013256073, + 0.5258598327636719 + ], + [ + 0.32326239347457886, + 0.5583849549293518 + ], + [ + 0.3380397856235504, + 0.5909103155136108 + ], + [ + 0.352817177772522, + 0.623435914516449 + ], + [ + 0.36759456992149353, + 0.6559617519378662 + ], + [ + 0.3823719322681427, + 0.6884878277778625 + ], + [ + 0.39714932441711426, + 0.721014142036438 + ], + [ + 0.41192665696144104, + 0.7535408139228821 + ], + [ + 0.4267039895057678, + 0.7860677242279053 + ], + [ + 0.4414812922477722, + 0.8185949325561523 + ], + [ + 0.4562585949897766, + 0.8511224389076233 + ], + [ + 0.47103583812713623, + 0.8836503028869629 + ], + [ + 0.48581308126449585, + 0.9161785244941711 + ], + [ + 0.5005902647972107, + 0.948707103729248 + ], + [ + 0.5153675079345703, + 0.9812359809875488 + ], + [ + 0.5301446914672852, + 1.0137652158737183 + ], + [ + 0.5449219346046448, + 1.046294927597046 + ], + [ + 0.5596991777420044, + 1.0788248777389526 + ], + [ + 0.5744765400886536, + 1.1113553047180176 + ], + [ + 0.5892539024353027, + 1.1438862085342407 + ], + [ + 0.6040313243865967, + 1.1764174699783325 + ], + [ + 0.6188088655471802, + 1.208949089050293 + ], + [ + 0.6335865259170532, + 1.2414813041687012 + ], + [ + 0.648364245891571, + 1.274013876914978 + ], + [ + 0.6631421446800232, + 1.3065470457077026 + ], + [ + 0.6779201030731201, + 1.3390806913375854 + ], + [ + 0.6926981806755066, + 1.371614933013916 + ], + [ + 0.7074640989303589, + 1.4041519165039062 + ], + [ + 0.7221670746803284, + 1.4367012977600098 + ], + [ + 0.7367299199104309, + 1.4692773818969727 + ], + [ + 0.7510751485824585, + 1.5018945932388306 + ], + [ + 0.7651242613792419, + 1.5345673561096191 + ], + [ + 0.7787973880767822, + 1.5673108100891113 + ], + [ + 0.7920125126838684, + 1.6001399755477905 + ], + [ + 0.8046852350234985, + 1.6330705881118774 + ], + [ + 0.8167278170585632, + 1.66611909866333 + ], + [ + 0.8280492424964905, + 1.6993021965026855 + ], + [ + 0.8385547399520874, + 1.7326377630233765 + ], + [ + 0.8481456637382507, + 1.766144037246704 + ], + [ + 0.856719434261322, + 1.7998400926589966 + ], + [ + 0.8641700744628906, + 1.8337457180023193 + ], + [ + 0.8703880906105042, + 1.8678812980651855 + ], + [ + 0.875261127948761, + 1.9022676944732666 + ], + [ + 0.8786745071411133, + 1.9369266033172607 + ], + [ + 0.8805115222930908, + 1.971879482269287 + ], + [ + 0.8806543946266174, + 2.0071487426757812 + ], + [ + 0.8789843320846558, + 2.0427565574645996 + ], + [ + 0.8753821849822998, + 2.0787250995635986 + ], + [ + 0.869838297367096, + 2.1150567531585693 + ], + [ + 0.8624524474143982, + 2.1517333984375 + ], + [ + 0.8533244729042053, + 2.1887366771698 + ], + [ + 0.8425538539886475, + 2.226048469543457 + ], + [ + 0.8302397131919861, + 2.263650894165039 + ], + [ + 0.816480278968811, + 2.301525831222534 + ], + [ + 0.8013728260993958, + 2.3396553993225098 + ], + [ + 0.7850133180618286, + 2.3780221939086914 + ], + [ + 0.7674961686134338, + 2.4166083335876465 + ], + [ + 0.7489144802093506, + 2.4553961753845215 + ], + [ + 0.7293595671653748, + 2.494368553161621 + ], + [ + 0.7089214324951172, + 2.533508062362671 + ], + [ + 0.6876888871192932, + 2.5727970600128174 + ], + [ + 0.665749728679657, + 2.6122183799743652 + ], + [ + 0.6431912779808044, + 2.651754379272461 + ], + [ + 0.6200692653656006, + 2.691394805908203 + ], + [ + 0.5964087247848511, + 2.731137275695801 + ], + [ + 0.5722349882125854, + 2.770979166030884 + ], + [ + 0.5475742816925049, + 2.810917615890503 + ], + [ + 0.5224538445472717, + 2.8509483337402344 + ], + [ + 0.49690231680870056, + 2.8910679817199707 + ], + [ + 0.47094982862472534, + 2.93127179145813 + ], + [ + 0.44462811946868896, + 2.971554756164551 + ], + [ + 0.4179706573486328, + 3.011911630630493 + ], + [ + 0.3910125195980072, + 3.0523364543914795 + ], + [ + 0.3637904226779938, + 3.0928235054016113 + ], + [ + 0.336342453956604, + 3.133366107940674 + ], + [ + 0.3087080717086792, + 3.173957586288452 + ], + [ + 0.2809275984764099, + 3.2145910263061523 + ], + [ + 0.25304216146469116, + 3.2552595138549805 + ], + [ + 0.2250932902097702, + 3.2959556579589844 + ], + [ + 0.19712261855602264, + 3.336672067642212 + ], + [ + 0.16917157173156738, + 3.3774020671844482 + ], + [ + 0.14128103852272034, + 3.418138027191162 + ], + [ + 0.11349108070135117, + 3.4588732719421387 + ], + [ + 0.08584070205688477, + 3.499600648880005 + ], + [ + 0.05836760997772217, + 3.540313243865967 + ], + [ + 0.031108075752854347, + 3.5810046195983887 + ], + [ + 0.004096813499927521, + 3.6216683387756348 + ], + [ + -0.022633060812950134, + 3.6622982025146484 + ], + [ + -0.049050018191337585, + 3.702888250350952 + ], + [ + -0.07512406259775162, + 3.7434325218200684 + ], + [ + -0.10082659870386124, + 3.7839250564575195 + ], + [ + -0.12613031268119812, + 3.8243606090545654 + ], + [ + -0.1510089486837387, + 3.8647332191467285 + ], + [ + -0.17543718218803406, + 3.9050378799438477 + ], + [ + -0.19939035177230835, + 3.9452688694000244 + ], + [ + -0.22284428775310516, + 3.9854207038879395 + ], + [ + -0.24577516317367554, + 4.025487899780273 + ], + [ + -0.26815930008888245, + 4.065464496612549 + ], + [ + -0.28985288739204407, + 4.1053338050842285 + ], + [ + -0.3105919361114502, + 4.145066261291504 + ], + [ + -0.33011239767074585, + 4.184632301330566 + ], + [ + -0.34815022349357605, + 4.224003314971924 + ], + [ + -0.3644413650035858, + 4.263148784637451 + ], + [ + -0.3787217438220978, + 4.302039623260498 + ], + [ + -0.3907274007797241, + 4.340645790100098 + ], + [ + -0.4001944959163666, + 4.378937721252441 + ], + [ + -0.40685927867889404, + 4.416884899139404 + ], + [ + -0.41045811772346497, + 4.4544572830200195 + ], + [ + -0.41072750091552734, + 4.491624355316162 + ], + [ + -0.40740400552749634, + 4.528356552124023 + ], + [ + -0.4004855453968048, + 4.564655303955078 + ], + [ + -0.3902314007282257, + 4.600553512573242 + ], + [ + -0.37690070271492004, + 4.636085033416748 + ], + [ + -0.3607523441314697, + 4.67128324508667 + ], + [ + -0.3420449197292328, + 4.706181049346924 + ], + [ + -0.32103657722473145, + 4.740812301635742 + ], + [ + -0.2979850471019745, + 4.775211334228516 + ], + [ + -0.2731475234031677, + 4.809412002563477 + ], + [ + -0.24678070843219757, + 4.843447685241699 + ], + [ + -0.21914079785346985, + 4.877353668212891 + ], + [ + -0.19048355519771576, + 4.911164283752441 + ], + [ + -0.16106447577476501, + 4.944913864135742 + ], + [ + -0.13102509081363678, + 4.978619575500488 + ], + [ + -0.10039319843053818, + 5.0122785568237305 + ], + [ + -0.06919693201780319, + 5.0458903312683105 + ], + [ + -0.03746507689356804, + 5.079452991485596 + ], + [ + -0.005227350629866123, + 5.1129655838012695 + ], + [ + 0.027485284954309464, + 5.146428108215332 + ], + [ + 0.06064034625887871, + 5.179840087890625 + ], + [ + 0.09420355409383774, + 5.213201522827148 + ], + [ + 0.12813864648342133, + 5.246513843536377 + ], + [ + 0.16240715980529785, + 5.279778003692627 + ], + [ + 0.19696833193302155, + 5.312995910644531 + ], + [ + 0.2317790538072586, + 5.3461689949035645 + ], + [ + 0.266793817281723, + 5.379299640655518 + ], + [ + 0.30196475982666016, + 5.412391662597656 + ], + [ + 0.3372417688369751, + 5.4454474449157715 + ], + [ + 0.37257257103919983, + 5.478470325469971 + ], + [ + 0.40790289640426636, + 5.511464595794678 + ], + [ + 0.4431767165660858, + 5.544434547424316 + ], + [ + 0.478336364030838, + 5.577383518218994 + ], + [ + 0.5133227705955505, + 5.610316753387451 + ], + [ + 0.5480756759643555, + 5.643238544464111 + ], + [ + 0.5825338363647461, + 5.676154136657715 + ], + [ + 0.6166353225708008, + 5.709067344665527 + ], + [ + 0.6503174901008606, + 5.741983413696289 + ], + [ + 0.6835171580314636, + 5.774907112121582 + ], + [ + 0.7161709666252136, + 5.8078436851501465 + ], + [ + 0.7482153177261353, + 5.840796947479248 + ], + [ + 0.7795863747596741, + 5.873773097991943 + ], + [ + 0.8102203011512756, + 5.906775951385498 + ], + [ + 0.8400532603263855, + 5.939810276031494 + ], + [ + 0.8690049648284912, + 5.9728803634643555 + ], + [ + 0.8969439268112183, + 6.005988121032715 + ], + [ + 0.9237036108970642, + 6.039134979248047 + ], + [ + 0.9491175413131714, + 6.072321891784668 + ], + [ + 0.9730191230773926, + 6.105550289154053 + ], + [ + 0.9952419996261597, + 6.138820171356201 + ], + [ + 1.0156195163726807, + 6.172133445739746 + ], + [ + 1.0339852571487427, + 6.205490589141846 + ], + [ + 1.0501729249954224, + 6.238892555236816 + ], + [ + 1.0640157461166382, + 6.272340774536133 + ], + [ + 1.075347661972046, + 6.305835723876953 + ], + [ + 1.084001898765564, + 6.339378356933594 + ], + [ + 1.0898123979568481, + 6.372969627380371 + ], + [ + 1.0927863121032715, + 6.406609058380127 + ], + [ + 1.093105435371399, + 6.440292835235596 + ], + [ + 1.090950846672058, + 6.474018096923828 + ], + [ + 1.0865041017532349, + 6.507782459259033 + ], + [ + 1.079946517944336, + 6.541581630706787 + ], + [ + 1.0714592933654785, + 6.575413227081299 + ], + [ + 1.0612238645553589, + 6.609274387359619 + ], + [ + 1.0494211912155151, + 6.643161773681641 + ], + [ + 1.036232590675354, + 6.677072525024414 + ], + [ + 1.0218391418457031, + 6.71100378036499 + ], + [ + 1.006421685218811, + 6.7449517250061035 + ], + [ + 0.9901613593101501, + 6.778914451599121 + ], + [ + 0.9732388854026794, + 6.812887668609619 + ], + [ + 0.9558353424072266, + 6.846869468688965 + ], + [ + 0.9380521178245544, + 6.880856990814209 + ], + [ + 0.9199115633964539, + 6.91485071182251 + ], + [ + 0.9014359712600708, + 6.948850154876709 + ], + [ + 0.8826476335525513, + 6.98285436630249 + ], + [ + 0.8635689616203308, + 7.016862869262695 + ], + [ + 0.8442226052284241, + 7.050876140594482 + ], + [ + 0.8246312141418457, + 7.084892749786377 + ], + [ + 0.8048177361488342, + 7.118912696838379 + ], + [ + 0.7848052978515625, + 7.15293550491333 + ], + [ + 0.7646171450614929, + 7.186960697174072 + ], + [ + 0.7442769408226013, + 7.220987796783447 + ], + [ + 0.7238084673881531, + 7.255016326904297 + ], + [ + 0.703235924243927, + 7.289045810699463 + ], + [ + 0.682583749294281, + 7.323075771331787 + ], + [ + 0.6618766784667969, + 7.357105731964111 + ], + [ + 0.6411397457122803, + 7.391135215759277 + ], + [ + 0.6203982830047607, + 7.425163269042969 + ], + [ + 0.5996780395507812, + 7.4591898918151855 + ], + [ + 0.5790049433708191, + 7.4932146072387695 + ], + [ + 0.5584054589271545, + 7.5272369384765625 + ], + [ + 0.5379061102867126, + 7.56125545501709 + ], + [ + 0.5175339579582214, + 7.595271110534668 + ], + [ + 0.4973162114620209, + 7.629281997680664 + ], + [ + 0.4772806167602539, + 7.663288116455078 + ], + [ + 0.457455039024353, + 7.697288990020752 + ], + [ + 0.43786779046058655, + 7.731284141540527 + ], + [ + 0.41854748129844666, + 7.765272617340088 + ], + [ + 0.3995230197906494, + 7.799253940582275 + ], + [ + 0.38082367181777954, + 7.833227634429932 + ], + [ + 0.3624790608882904, + 7.867193222045898 + ], + [ + 0.34451907873153687, + 7.901149749755859 + ], + [ + 0.32697397470474243, + 7.935096263885498 + ], + [ + 0.3098742961883545, + 7.969033241271973 + ], + [ + 0.2932509779930115, + 8.002959251403809 + ], + [ + 0.2771351933479309, + 8.036873817443848 + ], + [ + 0.2615584135055542, + 8.070775985717773 + ], + [ + 0.24655242264270782, + 8.10466480255127 + ], + [ + 0.23214924335479736, + 8.138541221618652 + ], + [ + 0.21838118135929108, + 8.172403335571289 + ], + [ + 0.20528072118759155, + 8.206250190734863 + ], + [ + 0.19288058578968048, + 8.240081787109375 + ], + [ + 0.18121366202831268, + 8.273897171020508 + ], + [ + 0.17031297087669373, + 8.307695388793945 + ], + [ + 0.1602116823196411, + 8.341476440429688 + ], + [ + 0.15094305574893951, + 8.375238418579102 + ], + [ + 0.14254039525985718, + 8.408982276916504 + ], + [ + 0.13503706455230713, + 8.442705154418945 + ], + [ + 0.12846647202968597, + 8.476408958435059 + ], + [ + 0.12282804399728775, + 8.510091781616211 + ], + [ + 0.11808725446462631, + 8.543754577636719 + ], + [ + 0.11420957744121552, + 8.577399253845215 + ], + [ + 0.11116043478250504, + 8.6110258102417 + ], + [ + 0.10890527069568634, + 8.644634246826172 + ], + [ + 0.10740949213504791, + 8.678226470947266 + ], + [ + 0.10663850605487823, + 8.711803436279297 + ], + [ + 0.1065577045083046, + 8.74536418914795 + ], + [ + 0.10713250190019608, + 8.778911590576172 + ], + [ + 0.10832829773426056, + 8.812445640563965 + ], + [ + 0.11011053621768951, + 8.845966339111328 + ], + [ + 0.112444669008255, + 8.879474639892578 + ], + [ + 0.11529617011547089, + 8.912972450256348 + ], + [ + 0.11863056570291519, + 8.946459770202637 + ], + [ + 0.12241341173648834, + 8.979937553405762 + ], + [ + 0.12661030888557434, + 9.013405799865723 + ], + [ + 0.1311868578195572, + 9.046866416931152 + ], + [ + 0.13610877096652985, + 9.080318450927734 + ], + [ + 0.14134173095226288, + 9.113764762878418 + ], + [ + 0.14685149490833282, + 9.147205352783203 + ], + [ + 0.15260380506515503, + 9.18064022064209 + ], + [ + 0.158564493060112, + 9.214071273803711 + ], + [ + 0.16469934582710266, + 9.24749755859375 + ], + [ + 0.17097420990467072, + 9.280921936035156 + ], + [ + 0.17735493183135986, + 9.314343452453613 + ], + [ + 0.1838073432445526, + 9.347764015197754 + ], + [ + 0.19029729068279266, + 9.381183624267578 + ], + [ + 0.19679751992225647, + 9.414603233337402 + ], + [ + 0.20329780876636505, + 9.448022842407227 + ], + [ + 0.2097981721162796, + 9.481443405151367 + ], + [ + 0.21629860997200012, + 9.514863014221191 + ], + [ + 0.22279909253120422, + 9.548283576965332 + ], + [ + 0.2292996346950531, + 9.581703186035156 + ], + [ + 0.23580022156238556, + 9.615123748779297 + ], + [ + 0.2423008531332016, + 9.648544311523438 + ], + [ + 0.24880154430866241, + 9.681964874267578 + ], + [ + 0.2553022503852844, + 9.715385437011719 + ], + [ + 0.2618030309677124, + 9.74880599975586 + ], + [ + 0.2683038115501404, + 9.7822265625 + ], + [ + 0.27480462193489075, + 9.815648078918457 + ], + [ + 0.2813054919242859, + 9.849068641662598 + ], + [ + 0.28780636191368103, + 9.882490158081055 + ], + [ + 0.29430726170539856, + 9.915910720825195 + ], + [ + 0.3008081614971161, + 9.949331283569336 + ], + [ + 0.307309091091156, + 9.982752799987793 + ], + [ + 0.3138100206851959, + 10.01617431640625 + ], + [ + 0.3203109800815582, + 10.04959487915039 + ], + [ + 0.32681193947792053, + 10.083016395568848 + ], + [ + 0.33331289887428284, + 10.116436958312988 + ], + [ + 0.33981388807296753, + 10.149858474731445 + ], + [ + 0.34631484746932983, + 10.183279991149902 + ], + [ + 0.3528158366680145, + 10.216700553894043 + ], + [ + 0.3593168258666992, + 10.2501220703125 + ], + [ + 0.3658177852630615, + 10.283543586730957 + ], + [ + 0.3723187744617462, + 10.316965103149414 + ], + [ + 0.3804450035095215, + 10.35874080657959 + ], + [ + 0.3853207528591156, + 10.383807182312012 + ] + ] + } +] diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/05_root_path/meta.json b/kimodo/assets/demo/examples/kimodo-soma-rp/05_root_path/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..9af10578584953b1d8cabcadbe5b081040abef7b --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/05_root_path/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person is casually walking forward slowly", + "duration": 10.0, + "num_samples": 1, + "seed": 42, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/05_root_path/motion.npz b/kimodo/assets/demo/examples/kimodo-soma-rp/05_root_path/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..41238bb35ebc4c83c427fb7d5572a0b893492b10 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/05_root_path/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5878d7958c62f19163d5fc95b4111e8b4180590291bfb1ad5f2fa9ece0930e6 +size 433998 diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/06_root_waypoints/constraints.json b/kimodo/assets/demo/examples/kimodo-soma-rp/06_root_waypoints/constraints.json new file mode 100644 index 0000000000000000000000000000000000000000..4a9c206492ca4b1539a9131092e568e641682098 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/06_root_waypoints/constraints.json @@ -0,0 +1,24 @@ +[ + { + "type": "root2d", + "frame_indices": [ + 0, + 90, + 180 + ], + "smooth_root_2d": [ + [ + 0.0, + -0.013232914730906487 + ], + [ + -1.1690130233764648, + 1.5332785844802856 + ], + [ + 0.738669753074646, + 1.4469488859176636 + ] + ] + } +] diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/06_root_waypoints/meta.json b/kimodo/assets/demo/examples/kimodo-soma-rp/06_root_waypoints/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..de8b8f537c5fd9ecb0c75dc5ef6b4d5340c2079f --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/06_root_waypoints/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person is doing a hip hop dance while moving around", + "duration": 6.033333333333333, + "num_samples": 1, + "seed": 42, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/06_root_waypoints/motion.npz b/kimodo/assets/demo/examples/kimodo-soma-rp/06_root_waypoints/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..bae3aa764eead743fb7a4c43c35a227c47764b87 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/06_root_waypoints/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9612c98041e35317fb89ed8821bb977abe044e5bc04e2232d6ba587dbeccb20 +size 262162 diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/07_mixed_constraints/constraints.json b/kimodo/assets/demo/examples/kimodo-soma-rp/07_mixed_constraints/constraints.json new file mode 100644 index 0000000000000000000000000000000000000000..944db28c7c47bc64c7111c59be8e1470d98e5e19 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/07_mixed_constraints/constraints.json @@ -0,0 +1,942 @@ +[ + { + "type": "fullbody", + "frame_indices": [ + 108 + ], + "local_joints_rot": [ + [ + [ + -0.035887543112039566, + -0.02776639349758625, + -0.005372282117605209 + ], + [ + 0.06515975296497345, + -0.010784560814499855, + 0.006556123960763216 + ], + [ + -0.06292378902435303, + -0.05156821012496948, + -0.009085050784051418 + ], + [ + 0.11570766568183899, + -0.0793282613158226, + -0.03867234289646149 + ], + [ + 0.09106606245040894, + 0.06571822613477707, + 0.002558206906542182 + ], + [ + -0.06086159870028496, + 0.10295507311820984, + 0.02592187374830246 + ], + [ + -0.15437740087509155, + 0.16596992313861847, + 0.009326435625553131 + ], + [ + -0.0005251984694041312, + 0.0018051519291475415, + -9.946066711563617e-05 + ], + [ + -0.184775248169899, + -0.064349465072155, + 0.00573313795030117 + ], + [ + -0.18454650044441223, + 0.068090058863163, + -0.005659883841872215 + ], + [ + 0.20501427352428436, + -0.14578332006931305, + -0.04773213341832161 + ], + [ + 0.26504039764404297, + -0.16855353116989136, + -1.0829373598098755 + ], + [ + 0.006512798834592104, + -0.6961542367935181, + -0.011537229642271996 + ], + [ + 0.07062757760286331, + 0.03925099968910217, + -0.027518808841705322 + ], + [ + 0.14896969497203827, + 0.29287680983543396, + -0.2919791340827942 + ], + [ + 0.009383739903569221, + 0.0508926659822464, + -1.056564450263977 + ], + [ + 0.11172245442867279, + 0.12029653787612915, + -0.12930497527122498 + ], + [ + -0.41130027174949646, + -0.5924108028411865, + -0.0006285393028520048 + ], + [ + 0.006594705395400524, + 0.4732210040092468, + -0.002528452081605792 + ], + [ + -0.32021215558052063, + -0.25638389587402344, + -0.3734903335571289 + ], + [ + 0.09024477005004883, + -0.2926441431045532, + 0.2660353481769562 + ], + [ + -0.09575983881950378, + -0.055268142372369766, + 0.8844737410545349 + ], + [ + -0.0118059441447258, + 0.07546520978212357, + 0.0746397078037262 + ], + [ + 0.8310757875442505, + -0.012923321686685085, + 0.004925338551402092 + ], + [ + 0.03474503755569458, + -0.23956389725208282, + -0.16712959110736847 + ], + [ + -0.09206951409578323, + -0.03187529370188713, + 0.027407124638557434 + ], + [ + -0.2677958309650421, + 0.11606352031230927, + 0.036957308650016785 + ], + [ + 0.394832044839859, + -0.0007178321247920394, + 0.0004849981633014977 + ], + [ + -0.09032224863767624, + -0.14483025670051575, + -0.015989331528544426 + ], + [ + -0.0217722300440073, + 0.01900928094983101, + -0.025495363399386406 + ] + ] + ], + "root_positions": [ + [ + -0.09470777958631516, + 0.9947724342346191, + -3.980208396911621 + ] + ], + "smooth_root_2d": [ + [ + -0.09470777958631516, + -3.980208396911621 + ] + ] + }, + { + "type": "root2d", + "frame_indices": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151 + ], + "smooth_root_2d": [ + [ + -0.022358937188982964, + 0.03532936051487923 + ], + [ + -0.024468135088682175, + -0.0013195642968639731 + ], + [ + -0.02657654881477356, + -0.037969205528497696 + ], + [ + -0.028683679178357124, + -0.07462010532617569 + ], + [ + -0.030789025127887726, + -0.11127285659313202 + ], + [ + -0.032892078161239624, + -0.14792808890342712 + ], + [ + -0.03499194607138634, + -0.184586301445961 + ], + [ + -0.03708736225962639, + -0.2212478667497635 + ], + [ + -0.03917701542377472, + -0.25791314244270325 + ], + [ + -0.04125956818461418, + -0.2945826053619385 + ], + [ + -0.04333365708589554, + -0.3312567174434662 + ], + [ + -0.045397885143756866, + -0.3679359555244446 + ], + [ + -0.04745082929730415, + -0.4046209156513214 + ], + [ + -0.04949106276035309, + -0.44131216406822205 + ], + [ + -0.05151714012026787, + -0.4780103266239166 + ], + [ + -0.05352761223912239, + -0.5147159695625305 + ], + [ + -0.05552104488015175, + -0.551429808139801 + ], + [ + -0.05749599635601044, + -0.5881525278091431 + ], + [ + -0.059451062232255936, + -0.6248847842216492 + ], + [ + -0.061384834349155426, + -0.6616273522377014 + ], + [ + -0.06329593807458878, + -0.6983808875083923 + ], + [ + -0.06518300622701645, + -0.7351461052894592 + ], + [ + -0.06704472005367279, + -0.7719237804412842 + ], + [ + -0.06887973845005035, + -0.8087146878242493 + ], + [ + -0.07068677246570587, + -0.8455195426940918 + ], + [ + -0.07246451079845428, + -0.8823391795158386 + ], + [ + -0.07421167194843292, + -0.9191742539405823 + ], + [ + -0.07592695951461792, + -0.9560256004333496 + ], + [ + -0.07760907709598541, + -0.9928940534591675 + ], + [ + -0.07925672084093094, + -1.029780387878418 + ], + [ + -0.08086856454610825, + -1.0666853189468384 + ], + [ + -0.0824432522058487, + -1.1036096811294556 + ], + [ + -0.08397942036390305, + -1.1405543088912964 + ], + [ + -0.08547566086053848, + -1.1775201559066772 + ], + [ + -0.08693055063486099, + -1.2145079374313354 + ], + [ + -0.08834262937307358, + -1.2515183687210083 + ], + [ + -0.08971039950847626, + -1.2885526418685913 + ], + [ + -0.09103234112262726, + -1.3256113529205322 + ], + [ + -0.09230689704418182, + -1.362695574760437 + ], + [ + -0.0935325101017952, + -1.3998061418533325 + ], + [ + -0.09470757842063904, + -1.4369438886642456 + ], + [ + -0.09583047777414322, + -1.4741098880767822 + ], + [ + -0.0968996062874794, + -1.5113049745559692 + ], + [ + -0.09791331738233566, + -1.548530101776123 + ], + [ + -0.09886999428272247, + -1.58578622341156 + ], + [ + -0.0997680053114891, + -1.6230742931365967 + ], + [ + -0.10060573369264603, + -1.6603953838348389 + ], + [ + -0.1013815775513649, + -1.6977503299713135 + ], + [ + -0.10209395736455917, + -1.7351402044296265 + ], + [ + -0.1027413085103035, + -1.7725658416748047 + ], + [ + -0.10332208126783371, + -1.8100284337997437 + ], + [ + -0.10383477061986923, + -1.8475286960601807 + ], + [ + -0.10427788645029068, + -1.8850678205490112 + ], + [ + -0.10464996099472046, + -1.9226467609405518 + ], + [ + -0.10494954138994217, + -1.9602664709091187 + ], + [ + -0.10517755895853043, + -1.997925877571106 + ], + [ + -0.10533731430768967, + -2.0356218814849854 + ], + [ + -0.10543208569288254, + -2.0733516216278076 + ], + [ + -0.10546516627073288, + -2.111111879348755 + ], + [ + -0.10543984919786453, + -2.148899555206299 + ], + [ + -0.10535937547683716, + -2.1867120265960693 + ], + [ + -0.10522699356079102, + -2.224546194076538 + ], + [ + -0.10504589974880219, + -2.262399435043335 + ], + [ + -0.10481927543878555, + -2.3002686500549316 + ], + [ + -0.10455025732517242, + -2.338151216506958 + ], + [ + -0.10424194484949112, + -2.376044511795044 + ], + [ + -0.10389743000268936, + -2.4139459133148193 + ], + [ + -0.10351976752281189, + -2.451852560043335 + ], + [ + -0.10311200469732285, + -2.4897620677948 + ], + [ + -0.10267717391252518, + -2.5276718139648438 + ], + [ + -0.10221832990646362, + -2.5655791759490967 + ], + [ + -0.10173854231834412, + -2.6034812927246094 + ], + [ + -0.10124091058969498, + -2.64137601852417 + ], + [ + -0.10072856396436691, + -2.67926025390625 + ], + [ + -0.100204698741436, + -2.7171311378479004 + ], + [ + -0.09967257082462311, + -2.754986047744751 + ], + [ + -0.09913549572229385, + -2.7928221225738525 + ], + [ + -0.09859687089920044, + -2.8306362628936768 + ], + [ + -0.09806016832590103, + -2.8684253692626953 + ], + [ + -0.09752892702817917, + -2.906186103820801 + ], + [ + -0.09700676798820496, + -2.943915367126465 + ], + [ + -0.09649737179279327, + -2.98160982131958 + ], + [ + -0.09600447863340378, + -3.019265651702881 + ], + [ + -0.09553186595439911, + -3.056879758834839 + ], + [ + -0.09508336335420609, + -3.0944483280181885 + ], + [ + -0.09466280788183212, + -3.131967782974243 + ], + [ + -0.09427405893802643, + -3.1694345474243164 + ], + [ + -0.09392096847295761, + -3.2068448066711426 + ], + [ + -0.09360739588737488, + -3.244194984436035 + ], + [ + -0.09333716332912445, + -3.2814812660217285 + ], + [ + -0.09311125427484512, + -3.3187034130096436 + ], + [ + -0.09292776882648468, + -3.355863571166992 + ], + [ + -0.0927848145365715, + -3.3929643630981445 + ], + [ + -0.09268050640821457, + -3.4300084114074707 + ], + [ + -0.09261301904916763, + -3.4669981002807617 + ], + [ + -0.09258053451776505, + -3.5039358139038086 + ], + [ + -0.09258133918046951, + -3.5408236980438232 + ], + [ + -0.09261377900838852, + -3.5776638984680176 + ], + [ + -0.0926763191819191, + -3.6144583225250244 + ], + [ + -0.09276753664016724, + -3.6512088775634766 + ], + [ + -0.09288612008094788, + -3.687917470932007 + ], + [ + -0.0930309146642685, + -3.7245850563049316 + ], + [ + -0.09320087730884552, + -3.7612133026123047 + ], + [ + -0.09339512139558792, + -3.7978031635284424 + ], + [ + -0.09361287951469421, + -3.8343558311462402 + ], + [ + -0.09385351091623306, + -3.8708720207214355 + ], + [ + -0.09411647915840149, + -3.9073524475097656 + ], + [ + -0.09440135210752487, + -3.9437978267669678 + ], + [ + -0.09470777958631516, + -3.980208396911621 + ], + [ + -0.09503547102212906, + -4.016584873199463 + ], + [ + -0.09538418799638748, + -4.052927494049072 + ], + [ + -0.09575372189283371, + -4.089236736297607 + ], + [ + -0.09614387899637222, + -4.125512599945068 + ], + [ + -0.0965544655919075, + -4.1617560386657715 + ], + [ + -0.09698529541492462, + -4.197966575622559 + ], + [ + -0.09743614494800568, + -4.234145641326904 + ], + [ + -0.09790677577257156, + -4.27029275894165 + ], + [ + -0.09839694201946259, + -4.306408882141113 + ], + [ + -0.09890634566545486, + -4.342494487762451 + ], + [ + -0.09943470358848572, + -4.378549575805664 + ], + [ + -0.09998169541358948, + -4.41457462310791 + ], + [ + -0.10054702311754227, + -4.450570583343506 + ], + [ + -0.10113038867712021, + -4.486537456512451 + ], + [ + -0.10173150897026062, + -4.522475242614746 + ], + [ + -0.1023501306772232, + -4.558384895324707 + ], + [ + -0.10298605263233185, + -4.594265937805176 + ], + [ + -0.10363911837339401, + -4.6301188468933105 + ], + [ + -0.10430921614170074, + -4.665942668914795 + ], + [ + -0.10499630123376846, + -4.701738357543945 + ], + [ + -0.10570038110017776, + -4.737504482269287 + ], + [ + -0.10642150044441223, + -4.7732415199279785 + ], + [ + -0.10715975612401962, + -4.808948040008545 + ], + [ + -0.10791526734828949, + -4.844624042510986 + ], + [ + -0.10868816822767258, + -4.880269527435303 + ], + [ + -0.10947857797145844, + -4.915882587432861 + ], + [ + -0.11028657108545303, + -4.9514641761779785 + ], + [ + -0.11111218482255936, + -4.98701286315918 + ], + [ + -0.11195536702871323, + -5.022529602050781 + ], + [ + -0.1128159612417221, + -5.058013439178467 + ], + [ + -0.11369368433952332, + -5.093465328216553 + ], + [ + -0.11458808928728104, + -5.128885746002197 + ], + [ + -0.11549859493970871, + -5.164275646209717 + ], + [ + -0.11642441153526306, + -5.199635982513428 + ], + [ + -0.11736457794904709, + -5.234969615936279 + ], + [ + -0.11831795424222946, + -5.270277500152588 + ], + [ + -0.11928320676088333, + -5.305562496185303 + ], + [ + -0.12025882303714752, + -5.340827941894531 + ], + [ + -0.12124315649271011, + -5.3760762214660645 + ], + [ + -0.12223441153764725, + -5.41131067276001 + ], + [ + -0.12323068082332611, + -5.446536064147949 + ], + [ + -0.12448007613420486, + -5.4905595779418945 + ], + [ + -0.1252303272485733, + -5.516972541809082 + ] + ] + } +] diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/07_mixed_constraints/meta.json b/kimodo/assets/demo/examples/kimodo-soma-rp/07_mixed_constraints/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..5d611eac9ea802d4f80dfd77f9dd00de1ac17f40 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/07_mixed_constraints/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A person walking backward points to the right side with their right hand", + "duration": 5.066666666666666, + "num_samples": 1, + "seed": 49, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/07_mixed_constraints/motion.npz b/kimodo/assets/demo/examples/kimodo-soma-rp/07_mixed_constraints/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..b0f464150bfb31ba9dd9847981b4dad44db03e11 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/07_mixed_constraints/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c120ec0d66f529b0c0d4e182868a28100f6b3fca619e36010b0a436b1ba0d5b +size 220286 diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/08_stylized_text/meta.json b/kimodo/assets/demo/examples/kimodo-soma-rp/08_stylized_text/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..869dd5cf8e47fa4880d13ebcfc6e0e94242e7ea1 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/08_stylized_text/meta.json @@ -0,0 +1,12 @@ +{ + "text": "A zombie with their left arm extended forward walks with an uneven gait at a slow pace.", + "duration": 4.033333333333333, + "num_samples": 1, + "seed": 42, + "diffusion_steps": 100, + "cfg": { + "enabled": true, + "text_weight": 2.0, + "constraint_weight": 2.0 + } +} diff --git a/kimodo/assets/demo/examples/kimodo-soma-rp/08_stylized_text/motion.npz b/kimodo/assets/demo/examples/kimodo-soma-rp/08_stylized_text/motion.npz new file mode 100644 index 0000000000000000000000000000000000000000..edbc3fa31acb60467003c47cb7c0d896f22f0518 --- /dev/null +++ b/kimodo/assets/demo/examples/kimodo-soma-rp/08_stylized_text/motion.npz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0687bf3f1dd49edf5530f8fe10c41a8df10b9dcdf6c3c3cd9d111bbb686ac355 +size 175522 diff --git a/kimodo/assets/demo/nvidia_logo.png b/kimodo/assets/demo/nvidia_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..d8dddbec2ea084c8011619f86f9f5716b84c2471 Binary files /dev/null and b/kimodo/assets/demo/nvidia_logo.png differ diff --git a/kimodo/assets/demo/nvidia_logo_dark.png b/kimodo/assets/demo/nvidia_logo_dark.png new file mode 100644 index 0000000000000000000000000000000000000000..f6a2bac42974127f9d99c2f03ca9248e44fbd95e Binary files /dev/null and b/kimodo/assets/demo/nvidia_logo_dark.png differ diff --git a/kimodo/assets/skeletons/g1skel34/joints.p b/kimodo/assets/skeletons/g1skel34/joints.p new file mode 100644 index 0000000000000000000000000000000000000000..4d5853a6e1ac0e6f2f854cd1a8893293612fa3ac Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/joints.p differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/head_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/head_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..401f822751c33dab53c6cee77eb62890aca5cd8f --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/head_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:005fb67fbd3eff94aa8bf4a6e83238174e9f91b6721f7111594322f223724411 +size 932784 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_ankle_pitch_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_ankle_pitch_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..69de8490184afc698f633e34bb74e65351bd4689 Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_ankle_pitch_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_ankle_roll_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_ankle_roll_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..cb69f65c6a8059fc6720cb393540f579b4d0c26e --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_ankle_roll_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4092af943141d4d9f74232f3cfa345afc6565f46a077793b8ae0e68b39dc33f +size 653384 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_elbow_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_elbow_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..1a96d99ba469960173129084ab6dd3bf8a732a71 Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_elbow_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_index_0_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_index_0_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..a87568d6835efb4f38efa6935c6ef30a2af633e0 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_index_0_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b35f2f77211d5a366f0b9a4e47c4ee35e536731266f1a34e9efa12db579b892 +size 475984 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_index_1_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_index_1_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..c6c91dd461805b2f0e89a1202e68f5b095b74a41 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_index_1_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e315ebc8a7a0cb98e033985b586b20d81cf8aa761181ae61ce56fcb14077a06 +size 1521784 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_middle_0_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_middle_0_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..a87568d6835efb4f38efa6935c6ef30a2af633e0 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_middle_0_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b35f2f77211d5a366f0b9a4e47c4ee35e536731266f1a34e9efa12db579b892 +size 475984 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_middle_1_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_middle_1_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..c6c91dd461805b2f0e89a1202e68f5b095b74a41 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_middle_1_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e315ebc8a7a0cb98e033985b586b20d81cf8aa761181ae61ce56fcb14077a06 +size 1521784 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_palm_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_palm_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..c8fcc3e4c3a864842b002eb94f6b692314ec97b8 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_palm_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23a486b75bd78a9bf03cec25d84d87f97f3dae038cf21a743b6d469b337e4004 +size 2140184 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_0_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_0_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..3028bb4d6e1ae3d30d2504259c08e4106bbacf63 Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_0_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_1_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_1_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..54f77542da4eab84d91035acd25405c09fb9d2e8 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_1_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:445c54a45bc13ce36001556f66bc0f49c83cb40321205ae4d676bb2874325684 +size 475984 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_2_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_2_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..3e4f124f2311fc11c2dcd7cec417da892ffd14c0 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hand_thumb_2_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d8dbe5085acfc213d21aa8b0782e89cd79084e9678f3a85fc7b04a86b029db5 +size 1521784 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_pitch_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_pitch_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..4cf7475bf6bbaef47d5b3f7644a34c73b3d64037 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_pitch_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4725168105ee768ee31638ef22b53f6be2d7641bfd7cfefe803488d884776fa4 +size 181684 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_roll_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_roll_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..585f6040c38571177d55519417a0f46ca7b29381 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_roll_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91f25922ee8a7c3152790051bebad17b4d9cd243569c38fe340285ff93a97acf +size 192184 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_yaw_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_yaw_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..b46a74134f90c568cbdfb8484eeb83a0503b4504 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_hip_yaw_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a16d88aa6ddac8083aa7ad55ed317bea44b1fa003d314fba88965b7ed0f3b55b +size 296284 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_knee_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_knee_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..2dcf84e4018ab9e4cff48cecdafe433601c48480 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_knee_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d92b9e3d3a636761150bb8025e32514c4602b91c7028d523ee42b3e632de477 +size 854884 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_rubber_hand.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_rubber_hand.STL new file mode 100644 index 0000000000000000000000000000000000000000..04a2fa22b7f60f2d00c64ec996b33a5219c2b633 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_rubber_hand.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cff2221a690fa69303f61fce68f2d155c1517b52efb6ca9262dd56e0bc6e70fe +size 2287484 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_pitch_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_pitch_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..926d9807f1cbf6ad0d41170904785eb30427c848 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_pitch_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0d1cfd02fcf0d42f95e678eeca33da3afbcc366ffba5c052847773ec4f31d52 +size 176784 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_roll_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_roll_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..4c6840b93ea15d1ba7b4dac0f18f94655867cab7 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_roll_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb9df21687773522598dc384f1a2945c7519f11cbc8bd372a49170316d6eee88 +size 400284 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_yaw_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_yaw_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..89b0e0661275ed832c674a46893239ad2e5bef85 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_shoulder_yaw_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aa97e9748e924336567992181f78c7cd0652fd52a4afcca3df6b2ef6f9e712e +size 249184 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_pitch_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_pitch_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..82cc224a8e41251d879502f9809e31d0988ec7f9 Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_pitch_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_roll_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_roll_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..2097ca3e5bbc0af985d150e5dcda4d8ba9bcf43f --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_roll_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edc387c9a0ba8c2237e9b296d32531426fabeb6f53e58df45c76106bca74148c +size 356184 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_roll_rubber_hand.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_roll_rubber_hand.STL new file mode 100644 index 0000000000000000000000000000000000000000..7c58819795c4cdfe406a008bfdbe7ea19d765497 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_roll_rubber_hand.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e81030abd023bd9e4a308ef376d814a2c12d684d8a7670c335bbd5cd7809c909 +size 3484884 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_yaw_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_yaw_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..692f4b07105fa0ec4ffd989dc60be9b2a33ac9c3 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/left_wrist_yaw_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83f8fb3a726bf9613d65dd14f0f447cb918c3c95b3938042a0c9c09749267d3b +size 318684 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/logo_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/logo_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..6c25961c594c714ada9d32e922250d326c121f4a --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/logo_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8571a0a19bc4916fa55f91449f51d5fdefd751000054865a842449429d5f155b +size 243384 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/pelvis.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/pelvis.STL new file mode 100644 index 0000000000000000000000000000000000000000..f98a88dbf6c7698791ba694e16cbcc180c40663c --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/pelvis.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ba6bbc888e630550140d3c26763f10206da8c8bd30ed886b8ede41c61f57a31 +size 1060884 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/pelvis_contour_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/pelvis_contour_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..8025bc070cd65550f345e9e97bc44a8ff9740dea --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/pelvis_contour_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cc5c2c7a312329e3feeb2b03d3fc09fc29705bd01864f6767e51be959662420 +size 1805184 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_ankle_pitch_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_ankle_pitch_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..e77d8a2fe1e5d56fac049833d254d6ffa4f6b350 Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_ankle_pitch_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_ankle_roll_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_ankle_roll_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..3c38507b9060eda2c40e5e2bb1b174ef5bda0390 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_ankle_roll_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b66222ea56653e627711b56d0a8949b4920da5df091da0ceb343f54e884e3a5 +size 653784 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_elbow_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_elbow_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..f259e3812efb9985d6463c04d3e8a4b53793a699 Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_elbow_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_index_0_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_index_0_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..5e2ef477e4666a8a26b9f02f89db5641cbbed0ab --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_index_0_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12ab4300c95e437e834f9aef772b3b431c671bf34338930b52ad11aef73bbd0d +size 475984 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_index_1_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_index_1_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..8572ae1c90f7c129b2d0a31c1ccfe23e02ed1820 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_index_1_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9c34efce4563cacdcfd29fc838a982976d40f5442c71219811dcbbf3923a33d +size 1521784 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_middle_0_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_middle_0_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..5e2ef477e4666a8a26b9f02f89db5641cbbed0ab --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_middle_0_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12ab4300c95e437e834f9aef772b3b431c671bf34338930b52ad11aef73bbd0d +size 475984 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_middle_1_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_middle_1_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..8572ae1c90f7c129b2d0a31c1ccfe23e02ed1820 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_middle_1_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9c34efce4563cacdcfd29fc838a982976d40f5442c71219811dcbbf3923a33d +size 1521784 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_palm_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_palm_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..fd2f7f0a8a9f75db8c45aef9e1a77c6b6224fd2a --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_palm_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86c0b231cc44477d64a6493e5a427ba16617a00738112dd187c652675b086fb9 +size 2140184 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_0_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_0_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..1cae7f18e16605cb9d6b1d1a0cf6e5c5c360a344 Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_0_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_1_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_1_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..c118de720dd2b5a467c397f9287f0ea01b2ad0a9 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_1_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a9a820da8dd10f298778b714f1364216e8a5976f4fd3a05689ea26327d44bf6 +size 475984 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_2_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_2_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..0979fb671e215be58c914ce22f03d47bdd0b6bf3 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hand_thumb_2_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f1bfb37668e8f61801c8d25f171fa1949e08666be86c67acad7e0079937cc45 +size 1521784 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_pitch_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_pitch_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..064085fca108bfc2a5a365403ceb6cdac4d5cd5f --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_pitch_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4f3c99d7f4a7d34eadbef9461fc66e3486cb5442db1ec50c86317d459f1a9c6 +size 181284 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_roll_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_roll_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..6544025eeb7dc7d09da549746ae28272e5bcf224 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_roll_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c254ef66a356f492947f360dd931965477b631e3fcc841f91ccc46d945d54f6 +size 192684 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_yaw_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_yaw_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..0ad7bee36419f85b490debe929957c6932df393f --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_hip_yaw_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e479c2936ca2057e9eb2f7dff6c189b7419d7b8484dea0b298cbb36a2a6aa668 +size 296284 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_knee_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_knee_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..65e8a708a7e86f1f242cd32e8959a560ea66c2d7 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_knee_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c4008449c9bbe701a6e2b557b7a252e90cf3a5abcf54cee46862b9a69f8ec8 +size 852284 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_rubber_hand.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_rubber_hand.STL new file mode 100644 index 0000000000000000000000000000000000000000..58148fb9e7307176d43f71da139781a77b63d156 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_rubber_hand.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99533b778bca6246144fa511bb9d4e555e075c641f2a0251e04372869cd99d67 +size 2192684 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_pitch_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_pitch_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..48a1c46009929341f2c312966acf81c3c733d7ec --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_pitch_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24cdb387e0128dfe602770a81c56cdce3a0181d34d039a11d1aaf8819b7b8c02 +size 176784 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_roll_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_roll_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..2a5d22f9e17a986ec8c958a6ff7668d65878a907 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_roll_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:962b97c48f9ce9e8399f45dd9522e0865d19aa9fd299406b2d475a8fc4a53e81 +size 401884 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_yaw_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_yaw_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..0882a567fc1a11f0cf821182154551058b0517f3 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_shoulder_yaw_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0b76489271da0c72461a344c9ffb0f0c6e64f019ea5014c1624886c442a2fe5 +size 249984 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_pitch_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_pitch_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..da194543c40df9d492abb0553c06cc614a78caae Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_pitch_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_roll_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_roll_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..77d23a77ad6ab659884db4d540e0bd9fd0675d0f --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_roll_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7ee9212ff5b94d6cb7f52bb1bbf3f352194d5b598acff74f4c77d340c5b344f +size 356084 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_roll_rubber_hand.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_roll_rubber_hand.STL new file mode 100644 index 0000000000000000000000000000000000000000..6f122afac362873186f291d520a255c575a4bf02 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_roll_rubber_hand.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0729aff1ac4356f9314de13a46906267642e58bc47f0d8a7f17f6590a6242ccf +size 3481584 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_yaw_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_yaw_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..77edbb4135bb5b9b64357a0226431fd665d794b2 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/right_wrist_yaw_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc9dece2d12509707e0057ba2e48df8f3d56db0c79410212963a25e8a50f61a6 +size 341484 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_L_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_L_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..cc2cbbf2ca4c72a20704c70fb61c80b7c4fa52aa --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_L_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82be7f93e85b3d303a1d1e1847e2c916939bd61c424ed1ebd28691ec33909dd1 +size 203584 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_L_rod_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_L_rod_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..6747f3f9341bd72b3803385c135c3ce750322e9d Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_L_rod_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_R_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_R_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..422ffe47ee75c14e5767f96ede5f506118c1054e --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_R_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54ded433a3a0c76027365856fdbd55215643de88846f7d436598a4071e682725 +size 203584 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_R_rod_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_R_rod_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..95cf415f72f1679a5867ca21a4af774f0b217cad Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_constraint_R_rod_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..e4fb87c4cc736c90970bb5f182981f0cf1544038 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e96d023f0368a4e3450b86ca5d4f10227d8141156a373e7da8cb3c93266523e0 +size 2232984 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link_23dof_rev_1_0.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link_23dof_rev_1_0.STL new file mode 100644 index 0000000000000000000000000000000000000000..edaf96a13e97536f92a114c7608a9f98ae0aba20 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link_23dof_rev_1_0.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cd0d56fde14b73c1623304684805029971c4f84b596f9914e823ca70a107fd2 +size 7825434 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link_rev_1_0.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link_rev_1_0.STL new file mode 100644 index 0000000000000000000000000000000000000000..836b9923b3edac7a1c06ecd2f11bfbbc0dedf7e0 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/torso_link_rev_1_0.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11ddb46f2098efbbd8816b1d65893632d8e78be936376c7cdcd6771899ccc723 +size 2570584 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_constraint_L.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_constraint_L.STL new file mode 100644 index 0000000000000000000000000000000000000000..6ec689bc8d8ca4a17fa3663f29d546837fef4ee2 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_constraint_L.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ebafdcb4de6871113f0ca2c356618d6e46b1d50f6c0bf9e37f47b9d8e100d99 +size 114684 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_constraint_R.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_constraint_R.STL new file mode 100644 index 0000000000000000000000000000000000000000..69fd76adbd96709fe00b68da0f0da7ef08b03d79 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_constraint_R.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:791902a291ffbd35ab97383b7b44ea5d975de7c80eef838797c970790b382ca9 +size 114684 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_roll_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_roll_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..65831abd2a64bc8c36e31016964413a3d2116725 Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_roll_link.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_roll_link_rev_1_0.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_roll_link_rev_1_0.STL new file mode 100644 index 0000000000000000000000000000000000000000..a64f330c592582dc31cdf38f4d08cfff06681c5f Binary files /dev/null and b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_roll_link_rev_1_0.STL differ diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_support_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_support_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..4a50f94fe3e60c6c128874488869779be9fef816 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_support_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fae9e1bb609848a1667d32eed8d6083ae443538a306843056a2a660f1b2926a +size 150484 diff --git a/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_yaw_link.STL b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_yaw_link.STL new file mode 100644 index 0000000000000000000000000000000000000000..c049debc4e0080a06ea95c2e7e9b74613cd42040 --- /dev/null +++ b/kimodo/assets/skeletons/g1skel34/meshes/g1/waist_yaw_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2883f20e03f09b669b5b4ce10677ee6b5191c0934b584d7cbaef2d0662856ffb +size 336284