Add files using upload-large-folder tool

.gitignore

@@ -0,0 +1,156 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.vscode
+dataset/dataset_TM_train_cb1_temp.py
+train_gpt_cnn_temp.py
+train_gpt_cnn_mask.py
+start.sh
+start_eval.sh
+config.json
+output_GPT_Final
+output_vqfinal
+output_transformer
+glove
+checkpoints
+dataset/HumanML3D
+dataset/KIT-ML
+output
+matrix_multi.py
+body_models
+render_final_diffuse.py
+render_final_mdm.py
+pretrained
+MDM
+Motiondiffusion
+Visualize_temp.py
+new.sh
+T2M_render
+render_final_t2m.py

GPT_eval_multi.py

@@ -0,0 +1,121 @@
+import os 
+import torch
+import numpy as np
+from torch.utils.tensorboard import SummaryWriter
+import json
+import clip
+
+import options.option_transformer as option_trans
+import models.vqvae as vqvae
+import utils.utils_model as utils_model
+import utils.eval_trans as eval_trans
+from dataset import dataset_TM_eval
+import models.t2m_trans as trans
+from options.get_eval_option import get_opt
+from models.evaluator_wrapper import EvaluatorModelWrapper
+import warnings
+warnings.filterwarnings('ignore')
+
+##### ---- Exp dirs ---- #####
+args = option_trans.get_args_parser()
+torch.manual_seed(args.seed)
+
+args.out_dir = os.path.join(args.out_dir, f'{args.exp_name}')
+os.makedirs(args.out_dir, exist_ok = True)
+
+##### ---- Logger ---- #####
+logger = utils_model.get_logger(args.out_dir)
+writer = SummaryWriter(args.out_dir)
+logger.info(json.dumps(vars(args), indent=4, sort_keys=True))
+
+from utils.word_vectorizer import WordVectorizer
+w_vectorizer = WordVectorizer('./glove', 'our_vab')
+val_loader = dataset_TM_eval.DATALoader(args.dataname, True, 32, w_vectorizer)
+
+dataset_opt_path = 'checkpoints/kit/Comp_v6_KLD005/opt.txt' if args.dataname == 'kit' else 'checkpoints/t2m/Comp_v6_KLD005/opt.txt'
+
+wrapper_opt = get_opt(dataset_opt_path, torch.device('cuda'))
+eval_wrapper = EvaluatorModelWrapper(wrapper_opt)
+
+##### ---- Network ---- #####
+
+## load clip model and datasets
+clip_model, clip_preprocess = clip.load("ViT-B/32", device=torch.device('cuda'), jit=False)  # Must set jit=False for training
+clip.model.convert_weights(clip_model)  # Actually this line is unnecessary since clip by default already on float16
+clip_model.eval()
+for p in clip_model.parameters():
+    p.requires_grad = False
+
+net = vqvae.HumanVQVAE(args, ## use args to define different parameters in different quantizers
+                       args.nb_code,
+                       args.code_dim,
+                       args.output_emb_width,
+                       args.down_t,
+                       args.stride_t,
+                       args.width,
+                       args.depth,
+                       args.dilation_growth_rate)
+
+
+trans_encoder = trans.Text2Motion_Transformer(num_vq=args.nb_code, 
+                                embed_dim=args.embed_dim_gpt, 
+                                clip_dim=args.clip_dim, 
+                                block_size=args.block_size, 
+                                num_layers=args.num_layers, 
+                                n_head=args.n_head_gpt, 
+                                drop_out_rate=args.drop_out_rate, 
+                                fc_rate=args.ff_rate)
+
+
+print ('loading checkpoint from {}'.format(args.resume_pth))
+ckpt = torch.load(args.resume_pth, map_location='cpu')
+net.load_state_dict(ckpt['net'], strict=True)
+net.eval()
+net.cuda()
+
+if args.resume_trans is not None:
+    print ('loading transformer checkpoint from {}'.format(args.resume_trans))
+    ckpt = torch.load(args.resume_trans, map_location='cpu')
+    trans_encoder.load_state_dict(ckpt['trans'], strict=True)
+trans_encoder.train()
+trans_encoder.cuda()
+
+
+fid = []
+div = []
+top1 = []
+top2 = []
+top3 = []
+matching = []
+multi = []
+repeat_time = 20
+
+        
+for i in range(repeat_time):
+    best_fid, best_iter, best_div, best_top1, best_top2, best_top3, best_matching, best_multi, writer, logger = eval_trans.evaluation_transformer_test(args.out_dir, val_loader, net, trans_encoder, logger, writer, 0, best_fid=1000, best_iter=0, best_div=100, best_top1=0, best_top2=0, best_top3=0, best_matching=100, best_multi=0, clip_model=clip_model, eval_wrapper=eval_wrapper, draw=False, savegif=False, save=False, savenpy=(i==0))
+    fid.append(best_fid)
+    div.append(best_div)
+    top1.append(best_top1)
+    top2.append(best_top2)
+    top3.append(best_top3)
+    matching.append(best_matching)
+    multi.append(best_multi)
+
+print('final result:')
+print('fid: ', sum(fid)/repeat_time)
+print('div: ', sum(div)/repeat_time)
+print('top1: ', sum(top1)/repeat_time)
+print('top2: ', sum(top2)/repeat_time)
+print('top3: ', sum(top3)/repeat_time)
+print('matching: ', sum(matching)/repeat_time)
+print('multi: ', sum(multi)/repeat_time)
+
+fid = np.array(fid)
+div = np.array(div)
+top1 = np.array(top1)
+top2 = np.array(top2)
+top3 = np.array(top3)
+matching = np.array(matching)
+multi = np.array(multi)
+msg_final = f"FID. {np.mean(fid):.3f}, conf. {np.std(fid)*1.96/np.sqrt(repeat_time):.3f}, Diversity. {np.mean(div):.3f}, conf. {np.std(div)*1.96/np.sqrt(repeat_time):.3f}, TOP1. {np.mean(top1):.3f}, conf. {np.std(top1)*1.96/np.sqrt(repeat_time):.3f}, TOP2. {np.mean(top2):.3f}, conf. {np.std(top2)*1.96/np.sqrt(repeat_time):.3f}, TOP3. {np.mean(top3):.3f}, conf. {np.std(top3)*1.96/np.sqrt(repeat_time):.3f}, Matching. {np.mean(matching):.3f}, conf. {np.std(matching)*1.96/np.sqrt(repeat_time):.3f}, Multi. {np.mean(multi):.3f}, conf. {np.std(multi)*1.96/np.sqrt(repeat_time):.3f}"
+logger.info(msg_final)

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 tencent
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,8 @@
+---
+title: T2M-GPT
+app_file: app.py
+sdk: gradio
+sdk_version: 3.50.2
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

VQ_eval.py

@@ -0,0 +1,95 @@
+import os
+import json
+
+import torch
+from torch.utils.tensorboard import SummaryWriter
+import numpy as np
+import models.vqvae as vqvae
+import options.option_vq as option_vq
+import utils.utils_model as utils_model
+from dataset import dataset_TM_eval
+import utils.eval_trans as eval_trans
+from options.get_eval_option import get_opt
+from models.evaluator_wrapper import EvaluatorModelWrapper
+import warnings
+warnings.filterwarnings('ignore')
+import numpy as np
+##### ---- Exp dirs ---- #####
+args = option_vq.get_args_parser()
+torch.manual_seed(args.seed)
+
+args.out_dir = os.path.join(args.out_dir, f'{args.exp_name}')
+os.makedirs(args.out_dir, exist_ok = True)
+
+##### ---- Logger ---- #####
+logger = utils_model.get_logger(args.out_dir)
+writer = SummaryWriter(args.out_dir)
+logger.info(json.dumps(vars(args), indent=4, sort_keys=True))
+
+
+from utils.word_vectorizer import WordVectorizer
+w_vectorizer = WordVectorizer('./glove', 'our_vab')
+
+
+dataset_opt_path = 'checkpoints/kit/Comp_v6_KLD005/opt.txt' if args.dataname == 'kit' else 'checkpoints/t2m/Comp_v6_KLD005/opt.txt'
+
+wrapper_opt = get_opt(dataset_opt_path, torch.device('cuda'))
+eval_wrapper = EvaluatorModelWrapper(wrapper_opt)
+
+
+##### ---- Dataloader ---- #####
+args.nb_joints = 21 if args.dataname == 'kit' else 22
+
+val_loader = dataset_TM_eval.DATALoader(args.dataname, True, 32, w_vectorizer, unit_length=2**args.down_t)
+
+##### ---- Network ---- #####
+net = vqvae.HumanVQVAE(args, ## use args to define different parameters in different quantizers
+                       args.nb_code,
+                       args.code_dim,
+                       args.output_emb_width,
+                       args.down_t,
+                       args.stride_t,
+                       args.width,
+                       args.depth,
+                       args.dilation_growth_rate,
+                       args.vq_act,
+                       args.vq_norm)
+
+if args.resume_pth : 
+    logger.info('loading checkpoint from {}'.format(args.resume_pth))
+    ckpt = torch.load(args.resume_pth, map_location='cpu')
+    net.load_state_dict(ckpt['net'], strict=True)
+net.train()
+net.cuda()
+
+fid = []
+div = []
+top1 = []
+top2 = []
+top3 = []
+matching = []
+repeat_time = 20
+for i in range(repeat_time):
+    best_fid, best_iter, best_div, best_top1, best_top2, best_top3, best_matching, writer, logger = eval_trans.evaluation_vqvae(args.out_dir, val_loader, net, logger, writer, 0, best_fid=1000, best_iter=0, best_div=100, best_top1=0, best_top2=0, best_top3=0, best_matching=100, eval_wrapper=eval_wrapper, draw=False, save=False, savenpy=(i==0))
+    fid.append(best_fid)
+    div.append(best_div)
+    top1.append(best_top1)
+    top2.append(best_top2)
+    top3.append(best_top3)
+    matching.append(best_matching)
+print('final result:')
+print('fid: ', sum(fid)/repeat_time)
+print('div: ', sum(div)/repeat_time)
+print('top1: ', sum(top1)/repeat_time)
+print('top2: ', sum(top2)/repeat_time)
+print('top3: ', sum(top3)/repeat_time)
+print('matching: ', sum(matching)/repeat_time)
+
+fid = np.array(fid)
+div = np.array(div)
+top1 = np.array(top1)
+top2 = np.array(top2)
+top3 = np.array(top3)
+matching = np.array(matching)
+msg_final = f"FID. {np.mean(fid):.3f}, conf. {np.std(fid)*1.96/np.sqrt(repeat_time):.3f}, Diversity. {np.mean(div):.3f}, conf. {np.std(div)*1.96/np.sqrt(repeat_time):.3f}, TOP1. {np.mean(top1):.3f}, conf. {np.std(top1)*1.96/np.sqrt(repeat_time):.3f}, TOP2. {np.mean(top2):.3f}, conf. {np.std(top2)*1.96/np.sqrt(repeat_time):.3f}, TOP3. {np.mean(top3):.3f}, conf. {np.std(top3)*1.96/np.sqrt(repeat_time):.3f}, Matching. {np.mean(matching):.3f}, conf. {np.std(matching)*1.96/np.sqrt(repeat_time):.3f}"
+logger.info(msg_final)

app.py

@@ -0,0 +1,379 @@
+import sys
+import os
+import OpenGL.GL as gl
+# os.environ["PYOPENGL_PLATFORM"] = "egl"
+os.environ["MESA_GL_VERSION_OVERRIDE"] = "4.1"
+# os.system('pip install /home/user/app/pyrender')
+
+sys.argv = ['VQ-Trans/GPT_eval_multi.py']
+os.chdir('VQ-Trans')
+
+sys.path.append('./VQ-Trans')
+sys.path.append('./pyrender')
+
+import options.option_transformer as option_trans
+from huggingface_hub import snapshot_download
+model_path = snapshot_download(repo_id="vumichien/T2M-GPT")
+
+args = option_trans.get_args_parser()
+
+args.dataname = 't2m'
+args.resume_pth = f'{model_path}/VQVAE/net_last.pth'
+args.resume_trans = f'{model_path}/VQTransformer_corruption05/net_best_fid.pth'
+args.down_t = 2
+args.depth = 3
+args.block_size = 51
+
+import clip
+import torch
+import numpy as np
+import models.vqvae as vqvae
+import models.t2m_trans as trans
+from utils.motion_process import recover_from_ric
+import visualization.plot_3d_global as plot_3d
+from models.rotation2xyz import Rotation2xyz
+import numpy as np
+from trimesh import Trimesh
+import gc
+
+import torch
+from visualize.simplify_loc2rot import joints2smpl
+import pyrender
+# import matplotlib.pyplot as plt
+
+import io
+import imageio
+from shapely import geometry
+import trimesh
+from pyrender.constants import RenderFlags
+import math
+# import ffmpeg
+# from PIL import Image
+import hashlib
+import gradio as gr
+import moviepy.editor as mp
+from datetime import datetime
+
+## load clip model and datasets
+is_cuda = torch.cuda.is_available()
+device = torch.device("cuda" if is_cuda else "cpu")
+print(device)
+clip_model, clip_preprocess = clip.load("ViT-B/32", device=device, jit=False, download_root='./')  # Must set jit=False for training
+
+if is_cuda:
+    clip.model.convert_weights(clip_model)
+    
+clip_model.eval()
+for p in clip_model.parameters():
+    p.requires_grad = False
+
+net = vqvae.HumanVQVAE(args, ## use args to define different parameters in different quantizers
+                       args.nb_code,
+                       args.code_dim,
+                       args.output_emb_width,
+                       args.down_t,
+                       args.stride_t,
+                       args.width,
+                       args.depth,
+                       args.dilation_growth_rate)
+
+
+trans_encoder = trans.Text2Motion_Transformer(num_vq=args.nb_code, 
+                                embed_dim=1024, 
+                                clip_dim=args.clip_dim, 
+                                block_size=args.block_size, 
+                                num_layers=9, 
+                                n_head=16, 
+                                drop_out_rate=args.drop_out_rate, 
+                                fc_rate=args.ff_rate)
+
+
+print('loading checkpoint from {}'.format(args.resume_pth))
+ckpt = torch.load(args.resume_pth, map_location='cpu')
+net.load_state_dict(ckpt['net'], strict=True)
+net.eval()
+    
+print('loading transformer checkpoint from {}'.format(args.resume_trans))
+ckpt = torch.load(args.resume_trans, map_location='cpu')
+trans_encoder.load_state_dict(ckpt['trans'], strict=True)
+trans_encoder.eval()
+
+mean = torch.from_numpy(np.load(f'{model_path}/meta/mean.npy'))
+std = torch.from_numpy(np.load(f'{model_path}/meta/std.npy'))
+
+if is_cuda:
+    net.cuda()
+    trans_encoder.cuda()
+    mean = mean.cuda()
+    std = std.cuda()
+
+def ensure_directory(path):
+    """Tạo thư mục nếu chưa tồn tại"""
+    if not os.path.exists(path):
+        os.makedirs(path)
+        print(f"Created directory: {path}")
+
+def get_output_path(output_dir, filename, extension):
+    """Tạo đường dẫn đầy đủ cho file output"""
+    ensure_directory(output_dir)
+    if not filename.endswith(extension):
+        filename += extension
+    return os.path.join(output_dir, filename)
+
+def render(motions, output_dir='output', filename='results', device_id=0):
+    """
+    Render motion với tùy chọn thư mục và tên file
+    
+    Args:
+        motions: Motion data
+        output_dir: Thư mục lưu kết quả (mặc định: 'output')
+        filename: Tên file không có extension (mặc định: 'results')
+        device_id: GPU device ID
+    """
+    frames, njoints, nfeats = motions.shape
+    MINS = motions.min(axis=0).min(axis=0)
+    MAXS = motions.max(axis=0).max(axis=0)
+
+    height_offset = MINS[1]
+    motions[:, :, 1] -= height_offset
+    trajec = motions[:, 0, [0, 2]]
+    is_cuda = torch.cuda.is_available()
+    j2s = joints2smpl(num_frames=frames, device_id=0, cuda=is_cuda)
+    rot2xyz = Rotation2xyz(device=device)
+    faces = rot2xyz.smpl_model.faces
+
+    # Tạo đường dẫn cho file .pt
+    pt_path = get_output_path(output_dir, f'{filename}_pred', '.pt')
+    
+    if not os.path.exists(pt_path): 
+        print(f'Running SMPLify, it may take a few minutes.')
+        motion_tensor, opt_dict = j2s.joint2smpl(motions)
+
+        vertices = rot2xyz(torch.tensor(motion_tensor).clone(), mask=None,
+                                        pose_rep='rot6d', translation=True, glob=True,
+                                        jointstype='vertices',
+                                        vertstrans=True)
+        vertices = vertices.detach().cpu()
+        torch.save(vertices, pt_path)
+    else:
+        vertices = torch.load(pt_path)
+    
+    frames = vertices.shape[3]
+    print(vertices.shape)
+    MINS = torch.min(torch.min(vertices[0], axis=0)[0], axis=1)[0]
+    MAXS = torch.max(torch.max(vertices[0], axis=0)[0], axis=1)[0]
+
+    out_list = []
+    
+    minx = MINS[0] - 0.5
+    maxx = MAXS[0] + 0.5
+    minz = MINS[2] - 0.5 
+    maxz = MAXS[2] + 0.5
+    polygon = geometry.Polygon([[minx, minz], [minx, maxz], [maxx, maxz], [maxx, minz]])
+    polygon_mesh = trimesh.creation.extrude_polygon(polygon, 1e-5)
+
+    vid = []
+    for i in range(frames):
+        if i % 10 == 0:
+            print(f"Processing frame {i}/{frames}")
+
+        mesh = Trimesh(vertices=vertices[0, :, :, i].squeeze().tolist(), faces=faces)
+
+        base_color = (0.11, 0.53, 0.8, 0.5)
+        material = pyrender.MetallicRoughnessMaterial(
+            metallicFactor=0.7,
+            alphaMode='OPAQUE',
+            baseColorFactor=base_color
+        )
+
+        mesh = pyrender.Mesh.from_trimesh(mesh, material=material)
+
+        polygon_mesh.visual.face_colors = [0, 0, 0, 0.21]
+        polygon_render = pyrender.Mesh.from_trimesh(polygon_mesh, smooth=False)
+
+        bg_color = [1, 1, 1, 0.8]
+        scene = pyrender.Scene(bg_color=bg_color, ambient_light=(0.4, 0.4, 0.4))
+        
+        sx, sy, tx, ty = [0.75, 0.75, 0, 0.10]
+
+        camera = pyrender.PerspectiveCamera(yfov=(np.pi / 3.0))
+        light = pyrender.DirectionalLight(color=[1,1,1], intensity=300)
+
+        scene.add(mesh)
+
+        c = np.pi / 2
+        scene.add(polygon_render, pose=np.array([[ 1, 0, 0, 0],
+        [ 0, np.cos(c), -np.sin(c), MINS[1].cpu().numpy()],
+        [ 0, np.sin(c), np.cos(c), 0],
+        [ 0, 0, 0, 1]]))
+
+        light_pose = np.eye(4)
+        light_pose[:3, 3] = [0, -1, 1]
+        scene.add(light, pose=light_pose.copy())
+
+        light_pose[:3, 3] = [0, 1, 1]
+        scene.add(light, pose=light_pose.copy())
+
+        light_pose[:3, 3] = [1, 1, 2]
+        scene.add(light, pose=light_pose.copy())
+
+        c = -np.pi / 6
+        scene.add(camera, pose=[[ 1, 0, 0, (minx+maxx).cpu().numpy()/2],
+                                [ 0, np.cos(c), -np.sin(c), 1.5],
+                                [ 0, np.sin(c), np.cos(c), max(4, minz.cpu().numpy()+(1.5-MINS[1].cpu().numpy())*2, (maxx-minx).cpu().numpy())],
+                                [ 0, 0, 0, 1]
+                                ])
+        
+        r = pyrender.OffscreenRenderer(960, 960)
+        color, _ = r.render(scene, flags=RenderFlags.RGBA)
+        vid.append(color)
+        r.delete()
+
+    out = np.stack(vid, axis=0)
+    
+    # Tạo đường dẫn cho file GIF và MP4
+    gif_path = get_output_path(output_dir, filename, '.gif')
+    mp4_path = get_output_path(output_dir, filename, '.mp4')
+    
+    imageio.mimwrite(gif_path, out, duration=50)
+    out_video = mp.VideoFileClip(gif_path)
+    out_video.write_videofile(mp4_path)
+    
+    print(f"Results saved to: {mp4_path}")
+    
+    del out, vertices
+    return mp4_path
+
+def predict(clip_text, method='fast', output_dir='output', filename=''):
+    """
+    Predict motion with custom output settings
+    
+    Args:
+        clip_text: Text prompt
+        method: 'fast' or 'slow'
+        output_dir: Output directory
+        filename: Custom filename (if empty, will use hash or timestamp)
+    """
+    gc.collect()
+    print('prompt text instruction: {}'.format(clip_text))
+    
+    # Tạo tên file nếu không được cung cấp
+    if not filename.strip():
+        if method == 'fast':
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            filename = f"motion_{timestamp}"
+        else:
+            filename = hashlib.md5(clip_text.encode()).hexdigest()
+    
+    # Xử lý text với CLIP
+    if torch.cuda.is_available():
+        text = clip.tokenize([clip_text], truncate=True).cuda()
+    else:
+        text = clip.tokenize([clip_text], truncate=True)
+    
+    feat_clip_text = clip_model.encode_text(text).float()
+    index_motion = trans_encoder.sample(feat_clip_text[0:1], False)
+    pred_pose = net.forward_decoder(index_motion)
+    pred_xyz = recover_from_ric((pred_pose*std+mean).float(), 22)
+    
+    if method == 'fast':
+        xyz = pred_xyz.reshape(1, -1, 22, 3)
+        
+        # Tạo đường dẫn cho fast method
+        gif_path = get_output_path(output_dir, filename, '.gif')
+        mp4_path = get_output_path(output_dir, filename, '.mp4')
+        
+        pose_vis = plot_3d.draw_to_batch(xyz.detach().cpu().numpy(), title_batch=None, outname=[gif_path])
+        out_video = mp.VideoFileClip(gif_path)
+        out_video.write_videofile(mp4_path)
+        
+        print(f"Fast render results saved to: {mp4_path}")
+        return mp4_path
+        
+    elif method == 'slow':
+        output_path = render(pred_xyz.detach().cpu().numpy().squeeze(axis=0), 
+                           output_dir=output_dir, 
+                           filename=filename, 
+                           device_id=0)
+        return output_path
+
+# ---- Gradio Layout -----
+video_out = gr.Video(label="Motion", mirror_webcam=False, interactive=False) 
+demo = gr.Blocks()
+demo.encrypt = False
+
+with demo:
+    gr.Markdown('''
+            <div>
+            <h1 style='text-align: center'>Generating Human Motion from Textual Descriptions (T2M-GPT)</h1>
+            This space uses <a href='https://mael-zys.github.io/T2M-GPT/' target='_blank'><b>T2M-GPT models</b></a> based on Vector Quantised-Variational AutoEncoder (VQ-VAE) and Generative Pre-trained Transformer (GPT) for human motion generation from textural descriptions🤗
+            </div>
+        ''')
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown('''
+            <figure>
+              <img src="https://huggingface.co/vumichien/T2M-GPT/resolve/main/demo_slow1.gif" alt="Demo Slow", width="425", height=480/>
+              <figcaption> a man starts off in an up right position with botg arms extended out by his sides, he then brings his arms down to his body and claps his hands together. after this he wals down amd the the left where he proceeds to sit on a seat
+              </figcaption>
+            </figure>
+            ''')
+        with gr.Column():
+            gr.Markdown('''
+            <figure>
+              <img src="https://huggingface.co/vumichien/T2M-GPT/resolve/main/demo_slow2.gif" alt="Demo Slow 2", width="425", height=480/>
+              <figcaption> a person puts their hands together, leans forwards slightly then swings the arms from right to left
+              </figcaption>
+            </figure>
+            ''')
+        with gr.Column():
+            gr.Markdown('''
+            <figure>
+              <img src="https://huggingface.co/vumichien/T2M-GPT/resolve/main/demo_slow3.gif" alt="Demo Slow 3", width="425", height=480/>
+              <figcaption> a man is practicing the waltz with a partner
+              </figcaption>
+            </figure>
+            ''')
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown('''
+            ### Generate human motion by **T2M-GPT**
+            ##### Step 1. Give prompt text describing human motion
+            ##### Step 2. Choose method to render output (Fast: Sketch skeleton; Slow: SMPL mesh)
+            ##### Step 3. Specify output directory and filename (optional)
+            ##### Step 4. Generate output and enjoy
+            ''')
+    with gr.Column():
+        with gr.Row():
+            text_prompt = gr.Textbox(label="Text prompt", lines=1, interactive=True)
+            method = gr.Dropdown(["slow", "fast"], label="Method", value="slow")
+        with gr.Row():
+            output_dir = gr.Textbox(label="Output Directory", value="output", interactive=True)
+            filename = gr.Textbox(label="Filename (without extension)", placeholder="Leave empty for auto-generated name", interactive=True)
+        with gr.Row():
+            generate_btn = gr.Button("Generate")
+            generate_btn.click(predict, [text_prompt, method, output_dir, filename], [video_out], api_name="generate")
+        with gr.Row():  
+            video_out.render()
+    with gr.Row(): 
+        gr.Markdown('''
+            ### You can test by following examples:
+            ''')
+    examples = gr.Examples(
+            examples=[
+                ["a person jogs in place, slowly at first, then increases speed. they then back up and squat down.", "slow", "output", "jogging_motion"],
+                ["a man steps forward and does a handstand", "slow", "output", "handstand_motion"],
+                ["a man rises from the ground, walks in a circle and sits back down on the ground", "slow", "output", "circle_walk"],
+                ["a man starts off in an up right position with botg arms extended out by his sides, he then brings his arms down to his body and claps his hands together. after this he wals down amd the the left where he proceeds to sit on a seat", "slow", "output", "clap_and_sit"],
+                ["a person puts their hands together, leans forwards slightly then swings the arms from right to left","slow", "output", "swing_arms"],
+                ["a man is practicing the waltz with a partner","slow", "output", "waltz_dance"],
+            ],
+            label="Examples", 
+            inputs=[text_prompt, method, output_dir, filename],
+            outputs=[video_out],
+            fn=predict,
+            cache_examples=True,
+        )
+
+        
+demo.launch(debug=True, server_name="0.0.0.0", server_port=8000, inbrowser=True, share=True)

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/README.md

@@ -0,0 +1,15 @@
+---
+title: Generate Human Motion
+emoji: 🏃
+colorFrom: green
+colorTo: yellow
+sdk: gradio
+sdk_version: 4.36.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+tags:
+- making-demos
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/VQ-Trans/visualization/plot_3d_global.py

@@ -0,0 +1,129 @@
+import torch 
+import matplotlib.pyplot as plt
+import numpy as np
+import io
+import matplotlib
+from mpl_toolkits.mplot3d.art3d import Poly3DCollection
+import mpl_toolkits.mplot3d.axes3d as p3
+from textwrap import wrap
+import imageio
+
+def plot_3d_motion(args, figsize=(10, 10), fps=120, radius=4):
+    matplotlib.use('Agg')
+    
+    
+    joints, out_name, title = args
+    
+    data = joints.copy().reshape(len(joints), -1, 3)
+    
+    nb_joints = joints.shape[1]
+    smpl_kinetic_chain = [[0, 11, 12, 13, 14, 15], [0, 16, 17, 18, 19, 20], [0, 1, 2, 3, 4], [3, 5, 6, 7], [3, 8, 9, 10]] if nb_joints == 21 else [[0, 2, 5, 8, 11], [0, 1, 4, 7, 10], [0, 3, 6, 9, 12, 15], [9, 14, 17, 19, 21], [9, 13, 16, 18, 20]]
+    limits = 1000 if nb_joints == 21 else 2
+    MINS = data.min(axis=0).min(axis=0)
+    MAXS = data.max(axis=0).max(axis=0)
+    colors = ['red', 'blue', 'black', 'red', 'blue',
+              'darkblue', 'darkblue', 'darkblue', 'darkblue', 'darkblue',
+              'darkred', 'darkred', 'darkred', 'darkred', 'darkred']
+    frame_number = data.shape[0]
+    #     print(data.shape)
+
+    height_offset = MINS[1]
+    data[:, :, 1] -= height_offset
+    trajec = data[:, 0, [0, 2]]
+
+    data[..., 0] -= data[:, 0:1, 0]
+    data[..., 2] -= data[:, 0:1, 2]
+
+    def update(index):
+
+        def init():
+            ax.set_xlim(-limits, limits)
+            ax.set_ylim(-limits, limits)
+            ax.set_zlim(0, limits)
+            ax.grid(b=False)
+        def plot_xzPlane(minx, maxx, miny, minz, maxz):
+            ## Plot a plane XZ
+            verts = [
+                [minx, miny, minz],
+                [minx, miny, maxz],
+                [maxx, miny, maxz],
+                [maxx, miny, minz]
+            ]
+            xz_plane = Poly3DCollection([verts])
+            xz_plane.set_facecolor((0.5, 0.5, 0.5, 0.5))
+            ax.add_collection3d(xz_plane)
+        fig = plt.figure(figsize=(480/96., 320/96.), dpi=96) if nb_joints == 21 else plt.figure(figsize=(10, 10), dpi=96)
+        if title is not None :
+            wraped_title = '\n'.join(wrap(title, 40))
+            fig.suptitle(wraped_title, fontsize=16)
+        ax = p3.Axes3D(fig)
+        
+        init()
+        
+        ax.lines = []
+        ax.collections = []
+        ax.view_init(elev=110, azim=-90)
+        ax.dist = 7.5
+        #         ax =
+        plot_xzPlane(MINS[0] - trajec[index, 0], MAXS[0] - trajec[index, 0], 0, MINS[2] - trajec[index, 1],
+                     MAXS[2] - trajec[index, 1])
+        #         ax.scatter(data[index, :22, 0], data[index, :22, 1], data[index, :22, 2], color='black', s=3)
+
+        if index > 1:
+            ax.plot3D(trajec[:index, 0] - trajec[index, 0], np.zeros_like(trajec[:index, 0]),
+                      trajec[:index, 1] - trajec[index, 1], linewidth=1.0,
+                      color='blue')
+        #             ax = plot_xzPlane(ax, MINS[0], MAXS[0], 0, MINS[2], MAXS[2])
+
+        for i, (chain, color) in enumerate(zip(smpl_kinetic_chain, colors)):
+            #             print(color)
+            if i < 5:
+                linewidth = 4.0
+            else:
+                linewidth = 2.0
+            ax.plot3D(data[index, chain, 0], data[index, chain, 1], data[index, chain, 2], linewidth=linewidth,
+                      color=color)
+        #         print(trajec[:index, 0].shape)
+
+        plt.axis('off')
+        ax.set_xticklabels([])
+        ax.set_yticklabels([])
+        ax.set_zticklabels([])
+    
+        if out_name is not None : 
+            plt.savefig(out_name, dpi=96)
+            plt.close()
+            
+        else : 
+            io_buf = io.BytesIO()
+            fig.savefig(io_buf, format='raw', dpi=96)
+            io_buf.seek(0)
+            # print(fig.bbox.bounds)
+            arr = np.reshape(np.frombuffer(io_buf.getvalue(), dtype=np.uint8),
+                                newshape=(int(fig.bbox.bounds[3]), int(fig.bbox.bounds[2]), -1))
+            io_buf.close()
+            plt.close()
+            return arr
+
+    out = []
+    for i in range(frame_number) : 
+        out.append(update(i))
+    out = np.stack(out, axis=0)
+    return torch.from_numpy(out)
+
+
+def draw_to_batch(smpl_joints_batch, title_batch=None, outname=None) : 
+    
+    batch_size = len(smpl_joints_batch)
+    out = []
+    for i in range(batch_size) : 
+        out.append(plot_3d_motion([smpl_joints_batch[i], None, title_batch[i] if title_batch is not None else None]))
+        if outname is not None:
+            imageio.mimsave(outname[i], np.array(out[-1]), fps=20)
+    out = torch.stack(out, axis=0)
+    return out
+    
+
+
+
+

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/VQ-Trans/visualize/joints2smpl/src/config.py

@@ -0,0 +1,40 @@
+import numpy as np
+
+# Map joints Name to SMPL joints idx
+JOINT_MAP = {
+'MidHip': 0,
+'LHip': 1, 'LKnee': 4, 'LAnkle': 7, 'LFoot': 10,
+'RHip': 2, 'RKnee': 5, 'RAnkle': 8, 'RFoot': 11,
+'LShoulder': 16, 'LElbow': 18, 'LWrist': 20, 'LHand': 22, 
+'RShoulder': 17, 'RElbow': 19, 'RWrist': 21, 'RHand': 23,
+'spine1': 3, 'spine2': 6, 'spine3': 9,  'Neck': 12, 'Head': 15,
+'LCollar':13, 'Rcollar' :14, 
+'Nose':24, 'REye':26,  'LEye':26,  'REar':27,  'LEar':28, 
+'LHeel': 31, 'RHeel': 34,
+'OP RShoulder': 17, 'OP LShoulder': 16,
+'OP RHip': 2, 'OP LHip': 1,
+'OP Neck': 12,
+}
+
+full_smpl_idx = range(24)
+key_smpl_idx = [0, 1, 4, 7,  2, 5, 8,  17, 19, 21,  16, 18, 20]
+
+
+AMASS_JOINT_MAP = {
+'MidHip': 0,
+'LHip': 1, 'LKnee': 4, 'LAnkle': 7, 'LFoot': 10,
+'RHip': 2, 'RKnee': 5, 'RAnkle': 8, 'RFoot': 11,
+'LShoulder': 16, 'LElbow': 18, 'LWrist': 20,  
+'RShoulder': 17, 'RElbow': 19, 'RWrist': 21, 
+'spine1': 3, 'spine2': 6, 'spine3': 9,  'Neck': 12, 'Head': 15,
+'LCollar':13, 'Rcollar' :14, 
+}
+amass_idx =       range(22)
+amass_smpl_idx =  range(22)
+
+
+SMPL_MODEL_DIR = "./body_models/"
+GMM_MODEL_DIR = "./visualize/joints2smpl/smpl_models/"
+SMPL_MEAN_FILE = "./visualize/joints2smpl/smpl_models/neutral_smpl_mean_params.h5"
+# for collsion 
+Part_Seg_DIR = "./visualize/joints2smpl/smpl_models/smplx_parts_segm.pkl"

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/VQ-Trans/visualize/joints2smpl/src/customloss.py

@@ -0,0 +1,222 @@
+import torch
+import torch.nn.functional as F
+from visualize.joints2smpl.src import config
+
+# Guassian
+def gmof(x, sigma):
+    """
+    Geman-McClure error function
+    """
+    x_squared = x ** 2
+    sigma_squared = sigma ** 2
+    return (sigma_squared * x_squared) / (sigma_squared + x_squared)
+
+# angle prior
+def angle_prior(pose):
+    """
+    Angle prior that penalizes unnatural bending of the knees and elbows
+    """
+    # We subtract 3 because pose does not include the global rotation of the model
+    return torch.exp(
+        pose[:, [55 - 3, 58 - 3, 12 - 3, 15 - 3]] * torch.tensor([1., -1., -1, -1.], device=pose.device)) ** 2
+
+
+def perspective_projection(points, rotation, translation,
+                           focal_length, camera_center):
+    """
+    This function computes the perspective projection of a set of points.
+    Input:
+        points (bs, N, 3): 3D points
+        rotation (bs, 3, 3): Camera rotation
+        translation (bs, 3): Camera translation
+        focal_length (bs,) or scalar: Focal length
+        camera_center (bs, 2): Camera center
+    """
+    batch_size = points.shape[0]
+    K = torch.zeros([batch_size, 3, 3], device=points.device)
+    K[:, 0, 0] = focal_length
+    K[:, 1, 1] = focal_length
+    K[:, 2, 2] = 1.
+    K[:, :-1, -1] = camera_center
+
+    # Transform points
+    points = torch.einsum('bij,bkj->bki', rotation, points)
+    points = points + translation.unsqueeze(1)
+
+    # Apply perspective distortion
+    projected_points = points / points[:, :, -1].unsqueeze(-1)
+
+    # Apply camera intrinsics
+    projected_points = torch.einsum('bij,bkj->bki', K, projected_points)
+
+    return projected_points[:, :, :-1]
+
+
+def body_fitting_loss(body_pose, betas, model_joints, camera_t, camera_center,
+                      joints_2d, joints_conf, pose_prior,
+                      focal_length=5000, sigma=100, pose_prior_weight=4.78,
+                      shape_prior_weight=5, angle_prior_weight=15.2,
+                      output='sum'):
+    """
+    Loss function for body fitting
+    """
+    batch_size = body_pose.shape[0]
+    rotation = torch.eye(3, device=body_pose.device).unsqueeze(0).expand(batch_size, -1, -1)
+
+    projected_joints = perspective_projection(model_joints, rotation, camera_t,
+                                              focal_length, camera_center)
+
+    # Weighted robust reprojection error
+    reprojection_error = gmof(projected_joints - joints_2d, sigma)
+    reprojection_loss = (joints_conf ** 2) * reprojection_error.sum(dim=-1)
+
+    # Pose prior loss
+    pose_prior_loss = (pose_prior_weight ** 2) * pose_prior(body_pose, betas)
+
+    # Angle prior for knees and elbows
+    angle_prior_loss = (angle_prior_weight ** 2) * angle_prior(body_pose).sum(dim=-1)
+
+    # Regularizer to prevent betas from taking large values
+    shape_prior_loss = (shape_prior_weight ** 2) * (betas ** 2).sum(dim=-1)
+
+    total_loss = reprojection_loss.sum(dim=-1) + pose_prior_loss + angle_prior_loss + shape_prior_loss
+
+    if output == 'sum':
+        return total_loss.sum()
+    elif output == 'reprojection':
+        return reprojection_loss
+
+
+# --- get camera fitting loss -----
+def camera_fitting_loss(model_joints, camera_t, camera_t_est, camera_center, 
+                        joints_2d, joints_conf,
+                        focal_length=5000, depth_loss_weight=100):
+    """
+    Loss function for camera optimization.
+    """
+    # Project model joints
+    batch_size = model_joints.shape[0]
+    rotation = torch.eye(3, device=model_joints.device).unsqueeze(0).expand(batch_size, -1, -1)
+    projected_joints = perspective_projection(model_joints, rotation, camera_t,
+                                              focal_length, camera_center)
+
+    # get the indexed four
+    op_joints = ['OP RHip', 'OP LHip', 'OP RShoulder', 'OP LShoulder']
+    op_joints_ind = [config.JOINT_MAP[joint] for joint in op_joints]
+    gt_joints = ['RHip', 'LHip', 'RShoulder', 'LShoulder']
+    gt_joints_ind = [config.JOINT_MAP[joint] for joint in gt_joints]
+
+    reprojection_error_op = (joints_2d[:, op_joints_ind] -
+                             projected_joints[:, op_joints_ind]) ** 2
+    reprojection_error_gt = (joints_2d[:, gt_joints_ind] -
+                             projected_joints[:, gt_joints_ind]) ** 2
+
+    # Check if for each example in the batch all 4 OpenPose detections are valid, otherwise use the GT detections
+    # OpenPose joints are more reliable for this task, so we prefer to use them if possible
+    is_valid = (joints_conf[:, op_joints_ind].min(dim=-1)[0][:, None, None] > 0).float()
+    reprojection_loss = (is_valid * reprojection_error_op + (1 - is_valid) * reprojection_error_gt).sum(dim=(1, 2))
+
+    # Loss that penalizes deviation from depth estimate
+    depth_loss = (depth_loss_weight ** 2) * (camera_t[:, 2] - camera_t_est[:, 2]) ** 2
+
+    total_loss = reprojection_loss + depth_loss
+    return total_loss.sum()
+
+
+
+ # #####--- body fitiing loss -----
+def body_fitting_loss_3d(body_pose, preserve_pose,
+                         betas, model_joints, camera_translation,
+                         j3d, pose_prior,
+                         joints3d_conf,
+                         sigma=100, pose_prior_weight=4.78*1.5,
+                         shape_prior_weight=5.0, angle_prior_weight=15.2,
+                         joint_loss_weight=500.0,
+                         pose_preserve_weight=0.0,
+                         use_collision=False,
+                         model_vertices=None, model_faces=None,
+                         search_tree=None,  pen_distance=None,  filter_faces=None,
+                         collision_loss_weight=1000
+                         ):
+    """
+    Loss function for body fitting
+    """
+    batch_size = body_pose.shape[0]
+
+    #joint3d_loss = (joint_loss_weight ** 2) * gmof((model_joints + camera_translation) - j3d, sigma).sum(dim=-1)
+    
+    joint3d_error = gmof((model_joints + camera_translation) - j3d, sigma)
+    
+    joint3d_loss_part = (joints3d_conf ** 2) * joint3d_error.sum(dim=-1)
+    joint3d_loss = ((joint_loss_weight ** 2) * joint3d_loss_part).sum(dim=-1)
+    
+    # Pose prior loss
+    pose_prior_loss = (pose_prior_weight ** 2) * pose_prior(body_pose, betas)
+    # Angle prior for knees and elbows
+    angle_prior_loss = (angle_prior_weight ** 2) * angle_prior(body_pose).sum(dim=-1)
+    # Regularizer to prevent betas from taking large values
+    shape_prior_loss = (shape_prior_weight ** 2) * (betas ** 2).sum(dim=-1)
+
+    collision_loss = 0.0
+    # Calculate the loss due to interpenetration
+    if use_collision:
+        triangles = torch.index_select(
+            model_vertices, 1,
+            model_faces).view(batch_size, -1, 3, 3)
+
+        with torch.no_grad():
+            collision_idxs = search_tree(triangles)
+
+        # Remove unwanted collisions
+        if filter_faces is not None:
+            collision_idxs = filter_faces(collision_idxs)
+
+        if collision_idxs.ge(0).sum().item() > 0:
+            collision_loss = torch.sum(collision_loss_weight * pen_distance(triangles, collision_idxs))
+    
+    pose_preserve_loss = (pose_preserve_weight ** 2) * ((body_pose - preserve_pose) ** 2).sum(dim=-1)
+
+    # print('joint3d_loss', joint3d_loss.shape)
+    # print('pose_prior_loss', pose_prior_loss.shape)
+    # print('angle_prior_loss', angle_prior_loss.shape)
+    # print('shape_prior_loss', shape_prior_loss.shape)
+    # print('collision_loss', collision_loss)
+    # print('pose_preserve_loss', pose_preserve_loss.shape)
+
+    total_loss = joint3d_loss + pose_prior_loss + angle_prior_loss + shape_prior_loss + collision_loss + pose_preserve_loss
+
+    return total_loss.sum()
+
+
+# #####--- get camera fitting loss -----
+def camera_fitting_loss_3d(model_joints, camera_t, camera_t_est,
+                           j3d, joints_category="orig", depth_loss_weight=100.0):
+    """
+    Loss function for camera optimization.
+    """
+    model_joints = model_joints + camera_t
+    # # get the indexed four
+    # op_joints = ['OP RHip', 'OP LHip', 'OP RShoulder', 'OP LShoulder']
+    # op_joints_ind = [config.JOINT_MAP[joint] for joint in op_joints]
+    #
+    # j3d_error_loss = (j3d[:, op_joints_ind] -
+    #                          model_joints[:, op_joints_ind]) ** 2
+
+    gt_joints = ['RHip', 'LHip', 'RShoulder', 'LShoulder']
+    gt_joints_ind = [config.JOINT_MAP[joint] for joint in gt_joints]
+    
+    if joints_category=="orig":
+        select_joints_ind = [config.JOINT_MAP[joint] for joint in gt_joints]
+    elif joints_category=="AMASS":
+        select_joints_ind = [config.AMASS_JOINT_MAP[joint] for joint in gt_joints]
+    else:
+        print("NO SUCH JOINTS CATEGORY!")
+
+    j3d_error_loss = (j3d[:, select_joints_ind] -
+                      model_joints[:, gt_joints_ind]) ** 2
+
+    # Loss that penalizes deviation from depth estimate
+    depth_loss = (depth_loss_weight**2) *  (camera_t - camera_t_est)**2
+
+    total_loss = j3d_error_loss +  depth_loss
+    return total_loss.sum()

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/VQ-Trans/visualize/joints2smpl/src/prior.py

@@ -0,0 +1,230 @@
+# -*- coding: utf-8 -*-
+
+# Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (MPG) is
+# holder of all proprietary rights on this computer program.
+# You can only use this computer program if you have closed
+# a license agreement with MPG or you get the right to use the computer
+# program from someone who is authorized to grant you that right.
+# Any use of the computer program without a valid license is prohibited and
+# liable to prosecution.
+#
+# Copyright©2019 Max-Planck-Gesellschaft zur Förderung
+# der Wissenschaften e.V. (MPG). acting on behalf of its Max Planck Institute
+# for Intelligent Systems. All rights reserved.
+#
+# Contact: ps-license@tuebingen.mpg.de
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import division
+
+import sys
+import os
+
+import time
+import pickle
+
+import numpy as np
+
+import torch
+import torch.nn as nn
+
+DEFAULT_DTYPE = torch.float32
+
+
+def create_prior(prior_type, **kwargs):
+    if prior_type == 'gmm':
+        prior = MaxMixturePrior(**kwargs)
+    elif prior_type == 'l2':
+        return L2Prior(**kwargs)
+    elif prior_type == 'angle':
+        return SMPLifyAnglePrior(**kwargs)
+    elif prior_type == 'none' or prior_type is None:
+        # Don't use any pose prior
+        def no_prior(*args, **kwargs):
+            return 0.0
+        prior = no_prior
+    else:
+        raise ValueError('Prior {}'.format(prior_type) + ' is not implemented')
+    return prior
+
+
+class SMPLifyAnglePrior(nn.Module):
+    def __init__(self, dtype=torch.float32, **kwargs):
+        super(SMPLifyAnglePrior, self).__init__()
+
+        # Indices for the roration angle of
+        # 55: left elbow,  90deg bend at -np.pi/2
+        # 58: right elbow, 90deg bend at np.pi/2
+        # 12: left knee,   90deg bend at np.pi/2
+        # 15: right knee,  90deg bend at np.pi/2
+        angle_prior_idxs = np.array([55, 58, 12, 15], dtype=np.int64)
+        angle_prior_idxs = torch.tensor(angle_prior_idxs, dtype=torch.long)
+        self.register_buffer('angle_prior_idxs', angle_prior_idxs)
+
+        angle_prior_signs = np.array([1, -1, -1, -1],
+                                     dtype=np.float32 if dtype == torch.float32
+                                     else np.float64)
+        angle_prior_signs = torch.tensor(angle_prior_signs,
+                                         dtype=dtype)
+        self.register_buffer('angle_prior_signs', angle_prior_signs)
+
+    def forward(self, pose, with_global_pose=False):
+        ''' Returns the angle prior loss for the given pose
+
+        Args:
+            pose: (Bx[23 + 1] * 3) torch tensor with the axis-angle
+            representation of the rotations of the joints of the SMPL model.
+        Kwargs:
+            with_global_pose: Whether the pose vector also contains the global
+            orientation of the SMPL model. If not then the indices must be
+            corrected.
+        Returns:
+            A sze (B) tensor containing the angle prior loss for each element
+            in the batch.
+        '''
+        angle_prior_idxs = self.angle_prior_idxs - (not with_global_pose) * 3
+        return torch.exp(pose[:, angle_prior_idxs] *
+                         self.angle_prior_signs).pow(2)
+
+
+class L2Prior(nn.Module):
+    def __init__(self, dtype=DEFAULT_DTYPE, reduction='sum', **kwargs):
+        super(L2Prior, self).__init__()
+
+    def forward(self, module_input, *args):
+        return torch.sum(module_input.pow(2))
+
+
+class MaxMixturePrior(nn.Module):
+
+    def __init__(self, prior_folder='prior',
+                 num_gaussians=6, dtype=DEFAULT_DTYPE, epsilon=1e-16,
+                 use_merged=True,
+                 **kwargs):
+        super(MaxMixturePrior, self).__init__()
+
+        if dtype == DEFAULT_DTYPE:
+            np_dtype = np.float32
+        elif dtype == torch.float64:
+            np_dtype = np.float64
+        else:
+            print('Unknown float type {}, exiting!'.format(dtype))
+            sys.exit(-1)
+
+        self.num_gaussians = num_gaussians
+        self.epsilon = epsilon
+        self.use_merged = use_merged
+        gmm_fn = 'gmm_{:02d}.pkl'.format(num_gaussians)
+
+        full_gmm_fn = os.path.join(prior_folder, gmm_fn)
+        if not os.path.exists(full_gmm_fn):
+            print('The path to the mixture prior "{}"'.format(full_gmm_fn) +
+                  ' does not exist, exiting!')
+            sys.exit(-1)
+
+        with open(full_gmm_fn, 'rb') as f:
+            gmm = pickle.load(f, encoding='latin1')
+
+        if type(gmm) == dict:
+            means = gmm['means'].astype(np_dtype)
+            covs = gmm['covars'].astype(np_dtype)
+            weights = gmm['weights'].astype(np_dtype)
+        elif 'sklearn.mixture.gmm.GMM' in str(type(gmm)):
+            means = gmm.means_.astype(np_dtype)
+            covs = gmm.covars_.astype(np_dtype)
+            weights = gmm.weights_.astype(np_dtype)
+        else:
+            print('Unknown type for the prior: {}, exiting!'.format(type(gmm)))
+            sys.exit(-1)
+
+        self.register_buffer('means', torch.tensor(means, dtype=dtype))
+
+        self.register_buffer('covs', torch.tensor(covs, dtype=dtype))
+
+        precisions = [np.linalg.inv(cov) for cov in covs]
+        precisions = np.stack(precisions).astype(np_dtype)
+
+        self.register_buffer('precisions',
+                             torch.tensor(precisions, dtype=dtype))
+
+        # The constant term:
+        sqrdets = np.array([(np.sqrt(np.linalg.det(c)))
+                            for c in gmm['covars']])
+        const = (2 * np.pi)**(69 / 2.)
+
+        nll_weights = np.asarray(gmm['weights'] / (const *
+                                                   (sqrdets / sqrdets.min())))
+        nll_weights = torch.tensor(nll_weights, dtype=dtype).unsqueeze(dim=0)
+        self.register_buffer('nll_weights', nll_weights)
+
+        weights = torch.tensor(gmm['weights'], dtype=dtype).unsqueeze(dim=0)
+        self.register_buffer('weights', weights)
+
+        self.register_buffer('pi_term',
+                             torch.log(torch.tensor(2 * np.pi, dtype=dtype)))
+
+        cov_dets = [np.log(np.linalg.det(cov.astype(np_dtype)) + epsilon)
+                    for cov in covs]
+        self.register_buffer('cov_dets',
+                             torch.tensor(cov_dets, dtype=dtype))
+
+        # The dimensionality of the random variable
+        self.random_var_dim = self.means.shape[1]
+
+    def get_mean(self):
+        ''' Returns the mean of the mixture '''
+        mean_pose = torch.matmul(self.weights, self.means)
+        return mean_pose
+
+    def merged_log_likelihood(self, pose, betas):
+        diff_from_mean = pose.unsqueeze(dim=1) - self.means
+
+        prec_diff_prod = torch.einsum('mij,bmj->bmi',
+                                      [self.precisions, diff_from_mean])
+        diff_prec_quadratic = (prec_diff_prod * diff_from_mean).sum(dim=-1)
+
+        curr_loglikelihood = 0.5 * diff_prec_quadratic - \
+            torch.log(self.nll_weights)
+        #  curr_loglikelihood = 0.5 * (self.cov_dets.unsqueeze(dim=0) +
+        #  self.random_var_dim * self.pi_term +
+        #  diff_prec_quadratic
+        #  ) - torch.log(self.weights)
+
+        min_likelihood, _ = torch.min(curr_loglikelihood, dim=1)
+        return min_likelihood
+
+    def log_likelihood(self, pose, betas, *args, **kwargs):
+        ''' Create graph operation for negative log-likelihood calculation
+        '''
+        likelihoods = []
+
+        for idx in range(self.num_gaussians):
+            mean = self.means[idx]
+            prec = self.precisions[idx]
+            cov = self.covs[idx]
+            diff_from_mean = pose - mean
+
+            curr_loglikelihood = torch.einsum('bj,ji->bi',
+                                              [diff_from_mean, prec])
+            curr_loglikelihood = torch.einsum('bi,bi->b',
+                                              [curr_loglikelihood,
+                                               diff_from_mean])
+            cov_term = torch.log(torch.det(cov) + self.epsilon)
+            curr_loglikelihood += 0.5 * (cov_term +
+                                         self.random_var_dim *
+                                         self.pi_term)
+            likelihoods.append(curr_loglikelihood)
+
+        log_likelihoods = torch.stack(likelihoods, dim=1)
+        min_idx = torch.argmin(log_likelihoods, dim=1)
+        weight_component = self.nll_weights[:, min_idx]
+        weight_component = -torch.log(weight_component)
+
+        return weight_component + log_likelihoods[:, min_idx]
+
+    def forward(self, pose, betas):
+        if self.use_merged:
+            return self.merged_log_likelihood(pose, betas)
+        else:
+            return self.log_likelihood(pose, betas)

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/VQ-Trans/visualize/joints2smpl/src/smplify.py

@@ -0,0 +1,279 @@
+import torch
+import os, sys
+import pickle
+import smplx
+import numpy as np
+
+sys.path.append(os.path.dirname(__file__))
+from customloss import (camera_fitting_loss, 
+                        body_fitting_loss, 
+                        camera_fitting_loss_3d,
+                        body_fitting_loss_3d, 
+                        )
+from prior import MaxMixturePrior
+from visualize.joints2smpl.src import config
+
+
+
+@torch.no_grad()
+def guess_init_3d(model_joints, 
+                  j3d, 
+                  joints_category="orig"):
+    """Initialize the camera translation via triangle similarity, by using the torso joints        .
+    :param model_joints: SMPL model with pre joints
+    :param j3d: 25x3 array of Kinect Joints
+    :returns: 3D vector corresponding to the estimated camera translation
+    """
+    # get the indexed four
+    gt_joints = ['RHip', 'LHip', 'RShoulder', 'LShoulder']
+    gt_joints_ind = [config.JOINT_MAP[joint] for joint in gt_joints]
+    
+    if joints_category=="orig":
+        joints_ind_category = [config.JOINT_MAP[joint] for joint in gt_joints]
+    elif joints_category=="AMASS":
+        joints_ind_category = [config.AMASS_JOINT_MAP[joint] for joint in gt_joints] 
+    else:
+        print("NO SUCH JOINTS CATEGORY!") 
+
+    sum_init_t = (j3d[:, joints_ind_category] - model_joints[:, gt_joints_ind]).sum(dim=1)
+    init_t = sum_init_t / 4.0
+    return init_t
+
+
+# SMPLIfy 3D
+class SMPLify3D():
+    """Implementation of SMPLify, use 3D joints."""
+
+    def __init__(self,
+                 smplxmodel,
+                 step_size=1e-2,
+                 batch_size=1,
+                 num_iters=100,
+                 use_collision=False,
+                 use_lbfgs=True,
+                 joints_category="orig",
+                 device=torch.device('cuda:0'),
+                 ):
+
+        # Store options
+        self.batch_size = batch_size
+        self.device = device
+        self.step_size = step_size
+
+        self.num_iters = num_iters
+        # --- choose optimizer
+        self.use_lbfgs = use_lbfgs
+        # GMM pose prior
+        self.pose_prior = MaxMixturePrior(prior_folder=config.GMM_MODEL_DIR,
+                                          num_gaussians=8,
+                                          dtype=torch.float32).to(device)
+        # collision part
+        self.use_collision = use_collision
+        if self.use_collision:
+            self.part_segm_fn = config.Part_Seg_DIR
+        
+        # reLoad SMPL-X model
+        self.smpl = smplxmodel
+
+        self.model_faces = smplxmodel.faces_tensor.view(-1)
+
+        # select joint joint_category
+        self.joints_category = joints_category
+        
+        if joints_category=="orig":
+            self.smpl_index = config.full_smpl_idx
+            self.corr_index = config.full_smpl_idx 
+        elif joints_category=="AMASS":
+            self.smpl_index = config.amass_smpl_idx
+            self.corr_index = config.amass_idx
+        else:
+            self.smpl_index = None 
+            self.corr_index = None
+            print("NO SUCH JOINTS CATEGORY!")
+
+    # ---- get the man function here ------
+    def __call__(self, init_pose, init_betas, init_cam_t, j3d, conf_3d=1.0, seq_ind=0):
+        """Perform body fitting.
+        Input:
+            init_pose: SMPL pose estimate
+            init_betas: SMPL betas estimate
+            init_cam_t: Camera translation estimate
+            j3d: joints 3d aka keypoints
+            conf_3d: confidence for 3d joints
+			seq_ind: index of the sequence
+        Returns:
+            vertices: Vertices of optimized shape
+            joints: 3D joints of optimized shape
+            pose: SMPL pose parameters of optimized shape
+            betas: SMPL beta parameters of optimized shape
+            camera_translation: Camera translation
+        """
+
+        # # # add the mesh inter-section to avoid
+        search_tree = None
+        pen_distance = None
+        filter_faces = None
+        
+        if self.use_collision:
+            from mesh_intersection.bvh_search_tree import BVH
+            import mesh_intersection.loss as collisions_loss
+            from mesh_intersection.filter_faces import FilterFaces
+
+            search_tree = BVH(max_collisions=8)
+
+            pen_distance = collisions_loss.DistanceFieldPenetrationLoss(
+                           sigma=0.5, point2plane=False, vectorized=True, penalize_outside=True)
+
+            if self.part_segm_fn:
+                # Read the part segmentation
+                part_segm_fn = os.path.expandvars(self.part_segm_fn)
+                with open(part_segm_fn, 'rb') as faces_parents_file:
+                    face_segm_data = pickle.load(faces_parents_file,  encoding='latin1')
+                faces_segm = face_segm_data['segm']
+                faces_parents = face_segm_data['parents']
+                # Create the module used to filter invalid collision pairs
+                filter_faces = FilterFaces(
+                    faces_segm=faces_segm, faces_parents=faces_parents,
+                    ign_part_pairs=None).to(device=self.device)
+                    
+                    
+        # Split SMPL pose to body pose and global orientation
+        body_pose = init_pose[:, 3:].detach().clone()
+        global_orient = init_pose[:, :3].detach().clone()
+        betas = init_betas.detach().clone()
+
+        # use guess 3d to get the initial
+        smpl_output = self.smpl(global_orient=global_orient,
+                                body_pose=body_pose,
+                                betas=betas)
+        model_joints = smpl_output.joints
+
+        init_cam_t = guess_init_3d(model_joints, j3d, self.joints_category).unsqueeze(1).detach()
+        camera_translation = init_cam_t.clone()
+        
+        preserve_pose = init_pose[:, 3:].detach().clone()
+       # -------------Step 1: Optimize camera translation and body orientation--------
+        # Optimize only camera translation and body orientation
+        body_pose.requires_grad = False
+        betas.requires_grad = False
+        global_orient.requires_grad = True
+        camera_translation.requires_grad = True
+
+        camera_opt_params = [global_orient, camera_translation]
+
+        if self.use_lbfgs:
+            camera_optimizer = torch.optim.LBFGS(camera_opt_params, max_iter=self.num_iters,
+                                                 lr=self.step_size, line_search_fn='strong_wolfe')
+            for i in range(10):
+                def closure():
+                    camera_optimizer.zero_grad()
+                    smpl_output = self.smpl(global_orient=global_orient,
+                                            body_pose=body_pose,
+                                            betas=betas)
+                    model_joints = smpl_output.joints
+                    # print('model_joints', model_joints.shape)
+                    # print('camera_translation', camera_translation.shape)
+                    # print('init_cam_t', init_cam_t.shape)
+                    # print('j3d', j3d.shape)
+                    loss = camera_fitting_loss_3d(model_joints, camera_translation,
+                                                  init_cam_t, j3d, self.joints_category)
+                    loss.backward()
+                    return loss
+
+                camera_optimizer.step(closure)
+        else:
+            camera_optimizer = torch.optim.Adam(camera_opt_params, lr=self.step_size, betas=(0.9, 0.999))
+
+            for i in range(20):
+                smpl_output = self.smpl(global_orient=global_orient,
+                                        body_pose=body_pose,
+                                        betas=betas)
+                model_joints = smpl_output.joints
+
+                loss = camera_fitting_loss_3d(model_joints[:, self.smpl_index], camera_translation,
+                                              init_cam_t,  j3d[:, self.corr_index], self.joints_category)
+                camera_optimizer.zero_grad()
+                loss.backward()
+                camera_optimizer.step()
+
+        # Fix camera translation after optimizing camera
+        # --------Step 2: Optimize body joints --------------------------
+        # Optimize only the body pose and global orientation of the body
+        body_pose.requires_grad = True
+        global_orient.requires_grad = True
+        camera_translation.requires_grad = True
+
+        # --- if we use the sequence, fix the shape
+        if seq_ind == 0:
+            betas.requires_grad = True
+            body_opt_params = [body_pose, betas, global_orient, camera_translation]
+        else:
+            betas.requires_grad = False
+            body_opt_params = [body_pose, global_orient, camera_translation]
+
+        if self.use_lbfgs:
+            body_optimizer = torch.optim.LBFGS(body_opt_params, max_iter=self.num_iters,
+                                               lr=self.step_size, line_search_fn='strong_wolfe')
+            for i in range(self.num_iters):
+                def closure():
+                    body_optimizer.zero_grad()
+                    smpl_output = self.smpl(global_orient=global_orient,
+                                            body_pose=body_pose,
+                                            betas=betas)
+                    model_joints = smpl_output.joints
+                    model_vertices = smpl_output.vertices
+
+                    loss = body_fitting_loss_3d(body_pose, preserve_pose, betas, model_joints[:, self.smpl_index], camera_translation,
+                                                j3d[:, self.corr_index], self.pose_prior,
+                                                joints3d_conf=conf_3d,
+                                                joint_loss_weight=600.0,
+                                                pose_preserve_weight=5.0,
+                                                use_collision=self.use_collision, 
+                                                model_vertices=model_vertices, model_faces=self.model_faces,
+                                                search_tree=search_tree, pen_distance=pen_distance, filter_faces=filter_faces)
+                    loss.backward()
+                    return loss
+
+                body_optimizer.step(closure)
+        else:
+            body_optimizer = torch.optim.Adam(body_opt_params, lr=self.step_size, betas=(0.9, 0.999))
+
+            for i in range(self.num_iters):
+                smpl_output = self.smpl(global_orient=global_orient,
+                                        body_pose=body_pose,
+                                        betas=betas)
+                model_joints = smpl_output.joints
+                model_vertices = smpl_output.vertices
+
+                loss = body_fitting_loss_3d(body_pose, preserve_pose, betas, model_joints[:, self.smpl_index], camera_translation,
+                                            j3d[:, self.corr_index], self.pose_prior,
+                                            joints3d_conf=conf_3d,
+                                            joint_loss_weight=600.0,
+                                            use_collision=self.use_collision, 
+                                            model_vertices=model_vertices, model_faces=self.model_faces,
+                                            search_tree=search_tree,  pen_distance=pen_distance,  filter_faces=filter_faces)
+                body_optimizer.zero_grad()
+                loss.backward()
+                body_optimizer.step()
+
+        # Get final loss value
+        with torch.no_grad():
+            smpl_output = self.smpl(global_orient=global_orient,
+                                    body_pose=body_pose,
+                                    betas=betas, return_full_pose=True)
+            model_joints = smpl_output.joints
+            model_vertices = smpl_output.vertices
+
+            final_loss = body_fitting_loss_3d(body_pose, preserve_pose, betas, model_joints[:, self.smpl_index], camera_translation,
+                                              j3d[:, self.corr_index], self.pose_prior,
+                                              joints3d_conf=conf_3d,
+                                              joint_loss_weight=600.0,
+                                              use_collision=self.use_collision, model_vertices=model_vertices, model_faces=self.model_faces,
+                                              search_tree=search_tree,  pen_distance=pen_distance,  filter_faces=filter_faces)
+
+        vertices = smpl_output.vertices.detach()
+        joints = smpl_output.joints.detach()
+        pose = torch.cat([global_orient, body_pose], dim=-1).detach()
+        betas = betas.detach()
+
+        return vertices, joints, pose, betas, camera_translation, final_loss

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/VQ-Trans/visualize/render_mesh.py

@@ -0,0 +1,33 @@
+import argparse
+import os
+from visualize import vis_utils
+import shutil
+from tqdm import tqdm
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_path", type=str, required=True, help='stick figure mp4 file to be rendered.')
+    parser.add_argument("--cuda", type=bool, default=True, help='')
+    parser.add_argument("--device", type=int, default=0, help='')
+    params = parser.parse_args()
+
+    assert params.input_path.endswith('.mp4')
+    parsed_name = os.path.basename(params.input_path).replace('.mp4', '').replace('sample', '').replace('rep', '')
+    sample_i, rep_i = [int(e) for e in parsed_name.split('_')]
+    npy_path = os.path.join(os.path.dirname(params.input_path), 'results.npy')
+    out_npy_path = params.input_path.replace('.mp4', '_smpl_params.npy')
+    assert os.path.exists(npy_path)
+    results_dir = params.input_path.replace('.mp4', '_obj')
+    if os.path.exists(results_dir):
+        shutil.rmtree(results_dir)
+    os.makedirs(results_dir)
+
+    npy2obj = vis_utils.npy2obj(npy_path, sample_i, rep_i,
+                                device=params.device, cuda=params.cuda)
+
+    print('Saving obj files to [{}]'.format(os.path.abspath(results_dir)))
+    for frame_i in tqdm(range(npy2obj.real_num_frames)):
+        npy2obj.save_obj(os.path.join(results_dir, 'frame{:03d}.obj'.format(frame_i)), frame_i)
+
+    print('Saving SMPL params to [{}]'.format(os.path.abspath(out_npy_path)))
+    npy2obj.save_npy(out_npy_path)

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/VQ-Trans/visualize/simplify_loc2rot.py

@@ -0,0 +1,131 @@
+import numpy as np
+import os
+import torch
+from visualize.joints2smpl.src import config
+import smplx
+import h5py
+from visualize.joints2smpl.src.smplify import SMPLify3D
+from tqdm import tqdm
+import utils.rotation_conversions as geometry
+import argparse
+
+
+class joints2smpl:
+
+    def __init__(self, num_frames, device_id, cuda=True):
+        self.device = torch.device("cuda:" + str(device_id) if cuda else "cpu")
+        # self.device = torch.device("cpu")
+        self.batch_size = num_frames
+        self.num_joints = 22  # for HumanML3D
+        self.joint_category = "AMASS"
+        self.num_smplify_iters = 150
+        self.fix_foot = False
+        print(config.SMPL_MODEL_DIR)
+        smplmodel = smplx.create(config.SMPL_MODEL_DIR,
+                                 model_type="smpl", gender="neutral", ext="pkl",
+                                 batch_size=self.batch_size).to(self.device)
+
+        # ## --- load the mean pose as original ----
+        smpl_mean_file = config.SMPL_MEAN_FILE
+
+        file = h5py.File(smpl_mean_file, 'r')
+        self.init_mean_pose = torch.from_numpy(file['pose'][:]).unsqueeze(0).repeat(self.batch_size, 1).float().to(self.device)
+        self.init_mean_shape = torch.from_numpy(file['shape'][:]).unsqueeze(0).repeat(self.batch_size, 1).float().to(self.device)
+        self.cam_trans_zero = torch.Tensor([0.0, 0.0, 0.0]).unsqueeze(0).to(self.device)
+        #
+
+        # # #-------------initialize SMPLify
+        self.smplify = SMPLify3D(smplxmodel=smplmodel,
+                            batch_size=self.batch_size,
+                            joints_category=self.joint_category,
+                            num_iters=self.num_smplify_iters,
+                            device=self.device)
+
+
+    def npy2smpl(self, npy_path):
+        out_path = npy_path.replace('.npy', '_rot.npy')
+        motions = np.load(npy_path, allow_pickle=True)[None][0]
+        # print_batch('', motions)
+        n_samples = motions['motion'].shape[0]
+        all_thetas = []
+        for sample_i in tqdm(range(n_samples)):
+            thetas, _ = self.joint2smpl(motions['motion'][sample_i].transpose(2, 0, 1))  # [nframes, njoints, 3]
+            all_thetas.append(thetas.cpu().numpy())
+        motions['motion'] = np.concatenate(all_thetas, axis=0)
+        print('motions', motions['motion'].shape)
+
+        print(f'Saving [{out_path}]')
+        np.save(out_path, motions)
+        exit()
+
+
+
+    def joint2smpl(self, input_joints, init_params=None):
+        _smplify = self.smplify # if init_params is None else self.smplify_fast
+        pred_pose = torch.zeros(self.batch_size, 72).to(self.device)
+        pred_betas = torch.zeros(self.batch_size, 10).to(self.device)
+        pred_cam_t = torch.zeros(self.batch_size, 3).to(self.device)
+        keypoints_3d = torch.zeros(self.batch_size, self.num_joints, 3).to(self.device)
+
+        # run the whole seqs
+        num_seqs = input_joints.shape[0]
+
+
+        # joints3d = input_joints[idx]  # *1.2 #scale problem [check first]
+        keypoints_3d = torch.Tensor(input_joints).to(self.device).float()
+
+        # if idx == 0:
+        if init_params is None:
+            pred_betas = self.init_mean_shape
+            pred_pose = self.init_mean_pose
+            pred_cam_t = self.cam_trans_zero
+        else:
+            pred_betas = init_params['betas']
+            pred_pose = init_params['pose']
+            pred_cam_t = init_params['cam']
+
+        if self.joint_category == "AMASS":
+            confidence_input = torch.ones(self.num_joints)
+            # make sure the foot and ankle
+            if self.fix_foot == True:
+                confidence_input[7] = 1.5
+                confidence_input[8] = 1.5
+                confidence_input[10] = 1.5
+                confidence_input[11] = 1.5
+        else:
+            print("Such category not settle down!")
+
+        new_opt_vertices, new_opt_joints, new_opt_pose, new_opt_betas, \
+        new_opt_cam_t, new_opt_joint_loss = _smplify(
+            pred_pose.detach(),
+            pred_betas.detach(),
+            pred_cam_t.detach(),
+            keypoints_3d,
+            conf_3d=confidence_input.to(self.device),
+            # seq_ind=idx
+        )
+
+        thetas = new_opt_pose.reshape(self.batch_size, 24, 3)
+        thetas = geometry.matrix_to_rotation_6d(geometry.axis_angle_to_matrix(thetas))  # [bs, 24, 6]
+        root_loc = torch.tensor(keypoints_3d[:, 0])  # [bs, 3]
+        root_loc = torch.cat([root_loc, torch.zeros_like(root_loc)], dim=-1).unsqueeze(1)  # [bs, 1, 6]
+        thetas = torch.cat([thetas, root_loc], dim=1).unsqueeze(0).permute(0, 2, 3, 1)  # [1, 25, 6, 196]
+
+        return thetas.clone().detach(), {'pose': new_opt_joints[0, :24].flatten().clone().detach(), 'betas': new_opt_betas.clone().detach(), 'cam': new_opt_cam_t.clone().detach()}
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--input_path", type=str, required=True, help='Blender file or dir with blender files')
+    parser.add_argument("--cuda", type=bool, default=True, help='')
+    parser.add_argument("--device", type=int, default=0, help='')
+    params = parser.parse_args()
+
+    simplify = joints2smpl(device_id=params.device, cuda=params.cuda)
+
+    if os.path.isfile(params.input_path) and params.input_path.endswith('.npy'):
+        simplify.npy2smpl(params.input_path)
+    elif os.path.isdir(params.input_path):
+        files = [os.path.join(params.input_path, f) for f in os.listdir(params.input_path) if f.endswith('.npy')]
+        for f in files:
+            simplify.npy2smpl(f)

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/VQ-Trans/visualize/vis_utils.py

@@ -0,0 +1,66 @@
+from model.rotation2xyz import Rotation2xyz
+import numpy as np
+from trimesh import Trimesh
+import os
+import torch
+from visualize.simplify_loc2rot import joints2smpl
+
+class npy2obj:
+    def __init__(self, npy_path, sample_idx, rep_idx, device=0, cuda=True):
+        self.npy_path = npy_path
+        self.motions = np.load(self.npy_path, allow_pickle=True)
+        if self.npy_path.endswith('.npz'):
+            self.motions = self.motions['arr_0']
+        self.motions = self.motions[None][0]
+        self.rot2xyz = Rotation2xyz(device='cpu')
+        self.faces = self.rot2xyz.smpl_model.faces
+        self.bs, self.njoints, self.nfeats, self.nframes = self.motions['motion'].shape
+        self.opt_cache = {}
+        self.sample_idx = sample_idx
+        self.total_num_samples = self.motions['num_samples']
+        self.rep_idx = rep_idx
+        self.absl_idx = self.rep_idx*self.total_num_samples + self.sample_idx
+        self.num_frames = self.motions['motion'][self.absl_idx].shape[-1]
+        self.j2s = joints2smpl(num_frames=self.num_frames, device_id=device, cuda=cuda)
+
+        if self.nfeats == 3:
+            print(f'Running SMPLify For sample [{sample_idx}], repetition [{rep_idx}], it may take a few minutes.')
+            motion_tensor, opt_dict = self.j2s.joint2smpl(self.motions['motion'][self.absl_idx].transpose(2, 0, 1))  # [nframes, njoints, 3]
+            self.motions['motion'] = motion_tensor.cpu().numpy()
+        elif self.nfeats == 6:
+            self.motions['motion'] = self.motions['motion'][[self.absl_idx]]
+        self.bs, self.njoints, self.nfeats, self.nframes = self.motions['motion'].shape
+        self.real_num_frames = self.motions['lengths'][self.absl_idx]
+
+        self.vertices = self.rot2xyz(torch.tensor(self.motions['motion']), mask=None,
+                                     pose_rep='rot6d', translation=True, glob=True,
+                                     jointstype='vertices',
+                                     # jointstype='smpl',  # for joint locations
+                                     vertstrans=True)
+        self.root_loc = self.motions['motion'][:, -1, :3, :].reshape(1, 1, 3, -1)
+        self.vertices += self.root_loc
+
+    def get_vertices(self, sample_i, frame_i):
+        return self.vertices[sample_i, :, :, frame_i].squeeze().tolist()
+
+    def get_trimesh(self, sample_i, frame_i):
+        return Trimesh(vertices=self.get_vertices(sample_i, frame_i),
+                       faces=self.faces)
+
+    def save_obj(self, save_path, frame_i):
+        mesh = self.get_trimesh(0, frame_i)
+        with open(save_path, 'w') as fw:
+            mesh.export(fw, 'obj')
+        return save_path
+    
+    def save_npy(self, save_path):
+        data_dict = {
+            'motion': self.motions['motion'][0, :, :, :self.real_num_frames],
+            'thetas': self.motions['motion'][0, :-1, :, :self.real_num_frames],
+            'root_translation': self.motions['motion'][0, -1, :3, :self.real_num_frames],
+            'faces': self.faces,
+            'vertices': self.vertices[0, :, :, :self.real_num_frames],
+            'text': self.motions['text'][0],
+            'length': self.real_num_frames,
+        }
+        np.save(save_path, data_dict)

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/app.py

@@ -0,0 +1,329 @@
+import sys
+import os
+import OpenGL.GL as gl
+os.environ["PYOPENGL_PLATFORM"] = "egl"
+os.environ["MESA_GL_VERSION_OVERRIDE"] = "4.1"
+os.system('pip install /home/user/app/pyrender')
+
+sys.argv = ['VQ-Trans/GPT_eval_multi.py']
+os.chdir('VQ-Trans')
+
+sys.path.append('/home/user/app/VQ-Trans')
+sys.path.append('/home/user/app/pyrender')
+
+import options.option_transformer as option_trans
+from huggingface_hub import snapshot_download
+model_path = snapshot_download(repo_id="vumichien/T2M-GPT")
+
+args = option_trans.get_args_parser()
+
+args.dataname = 't2m'
+args.resume_pth = f'{model_path}/VQVAE/net_last.pth'
+args.resume_trans = f'{model_path}/VQTransformer_corruption05/net_best_fid.pth'
+args.down_t = 2
+args.depth = 3
+args.block_size = 51
+
+import clip
+import torch
+import numpy as np
+import models.vqvae as vqvae
+import models.t2m_trans as trans
+from utils.motion_process import recover_from_ric
+import visualization.plot_3d_global as plot_3d
+from models.rotation2xyz import Rotation2xyz
+import numpy as np
+from trimesh import Trimesh
+import gc
+
+import torch
+from visualize.simplify_loc2rot import joints2smpl
+import pyrender
+# import matplotlib.pyplot as plt
+
+import io
+import imageio
+from shapely import geometry
+import trimesh
+from pyrender.constants import RenderFlags
+import math
+# import ffmpeg
+# from PIL import Image
+import hashlib
+import gradio as gr
+import moviepy.editor as mp
+
+## load clip model and datasets
+is_cuda = torch.cuda.is_available()
+device = torch.device("cuda" if is_cuda else "cpu")
+print(device)
+clip_model, clip_preprocess = clip.load("ViT-B/32", device=device, jit=False, download_root='./')  # Must set jit=False for training
+
+if is_cuda:
+    clip.model.convert_weights(clip_model)
+    
+clip_model.eval()
+for p in clip_model.parameters():
+    p.requires_grad = False
+
+net = vqvae.HumanVQVAE(args, ## use args to define different parameters in different quantizers
+                       args.nb_code,
+                       args.code_dim,
+                       args.output_emb_width,
+                       args.down_t,
+                       args.stride_t,
+                       args.width,
+                       args.depth,
+                       args.dilation_growth_rate)
+
+
+trans_encoder = trans.Text2Motion_Transformer(num_vq=args.nb_code, 
+                                embed_dim=1024, 
+                                clip_dim=args.clip_dim, 
+                                block_size=args.block_size, 
+                                num_layers=9, 
+                                n_head=16, 
+                                drop_out_rate=args.drop_out_rate, 
+                                fc_rate=args.ff_rate)
+
+
+print('loading checkpoint from {}'.format(args.resume_pth))
+ckpt = torch.load(args.resume_pth, map_location='cpu')
+net.load_state_dict(ckpt['net'], strict=True)
+net.eval()
+    
+print('loading transformer checkpoint from {}'.format(args.resume_trans))
+ckpt = torch.load(args.resume_trans, map_location='cpu')
+trans_encoder.load_state_dict(ckpt['trans'], strict=True)
+trans_encoder.eval()
+
+mean = torch.from_numpy(np.load(f'{model_path}/meta/mean.npy'))
+std = torch.from_numpy(np.load(f'{model_path}/meta/std.npy'))
+
+if is_cuda:
+    net.cuda()
+    trans_encoder.cuda()
+    mean = mean.cuda()
+    std = std.cuda()
+
+def render(motions, device_id=0, name='test_vis'):
+    frames, njoints, nfeats = motions.shape
+    MINS = motions.min(axis=0).min(axis=0)
+    MAXS = motions.max(axis=0).max(axis=0)
+
+    height_offset = MINS[1]
+    motions[:, :, 1] -= height_offset
+    trajec = motions[:, 0, [0, 2]]
+    is_cuda = torch.cuda.is_available()
+    # device = torch.device("cuda" if is_cuda else "cpu")
+    j2s = joints2smpl(num_frames=frames, device_id=0, cuda=is_cuda)
+    rot2xyz = Rotation2xyz(device=device)
+    faces = rot2xyz.smpl_model.faces
+
+    if not os.path.exists(f'output/{name}_pred.pt'): 
+        print(f'Running SMPLify, it may take a few minutes.')
+        motion_tensor, opt_dict = j2s.joint2smpl(motions)  # [nframes, njoints, 3]
+
+        vertices = rot2xyz(torch.tensor(motion_tensor).clone(), mask=None,
+                                        pose_rep='rot6d', translation=True, glob=True,
+                                        jointstype='vertices',
+                                        vertstrans=True)
+        vertices = vertices.detach().cpu()
+        torch.save(vertices, f'output/{name}_pred.pt')
+    else:
+        vertices = torch.load(f'output/{name}_pred.pt')
+    frames = vertices.shape[3] # shape: 1, nb_frames, 3, nb_joints
+    print(vertices.shape)
+    MINS = torch.min(torch.min(vertices[0], axis=0)[0], axis=1)[0]
+    MAXS = torch.max(torch.max(vertices[0], axis=0)[0], axis=1)[0]
+
+    out_list = []
+    
+    minx = MINS[0] - 0.5
+    maxx = MAXS[0] + 0.5
+    minz = MINS[2] - 0.5 
+    maxz = MAXS[2] + 0.5
+    polygon = geometry.Polygon([[minx, minz], [minx, maxz], [maxx, maxz], [maxx, minz]])
+    polygon_mesh = trimesh.creation.extrude_polygon(polygon, 1e-5)
+
+    vid = []
+    for i in range(frames):
+        if i % 10 == 0:
+            print(i)
+
+        mesh = Trimesh(vertices=vertices[0, :, :, i].squeeze().tolist(), faces=faces)
+
+        base_color = (0.11, 0.53, 0.8, 0.5)
+        ## OPAQUE rendering without alpha
+        ## BLEND rendering consider alpha 
+        material = pyrender.MetallicRoughnessMaterial(
+            metallicFactor=0.7,
+            alphaMode='OPAQUE',
+            baseColorFactor=base_color
+        )
+
+
+        mesh = pyrender.Mesh.from_trimesh(mesh, material=material)
+
+        polygon_mesh.visual.face_colors = [0, 0, 0, 0.21]
+        polygon_render = pyrender.Mesh.from_trimesh(polygon_mesh, smooth=False)
+
+        bg_color = [1, 1, 1, 0.8]
+        scene = pyrender.Scene(bg_color=bg_color, ambient_light=(0.4, 0.4, 0.4))
+        
+        sx, sy, tx, ty = [0.75, 0.75, 0, 0.10]
+
+        camera = pyrender.PerspectiveCamera(yfov=(np.pi / 3.0))
+
+        light = pyrender.DirectionalLight(color=[1,1,1], intensity=300)
+
+        scene.add(mesh)
+
+        c = np.pi / 2
+
+        scene.add(polygon_render, pose=np.array([[ 1, 0, 0, 0],
+
+        [ 0, np.cos(c), -np.sin(c), MINS[1].cpu().numpy()],
+
+        [ 0, np.sin(c), np.cos(c), 0],
+
+        [ 0, 0, 0, 1]]))
+
+        light_pose = np.eye(4)
+        light_pose[:3, 3] = [0, -1, 1]
+        scene.add(light, pose=light_pose.copy())
+
+        light_pose[:3, 3] = [0, 1, 1]
+        scene.add(light, pose=light_pose.copy())
+
+        light_pose[:3, 3] = [1, 1, 2]
+        scene.add(light, pose=light_pose.copy())
+
+
+        c = -np.pi / 6
+
+        scene.add(camera, pose=[[ 1, 0, 0, (minx+maxx).cpu().numpy()/2],
+
+                                [ 0, np.cos(c), -np.sin(c), 1.5],
+
+                                [ 0, np.sin(c), np.cos(c), max(4, minz.cpu().numpy()+(1.5-MINS[1].cpu().numpy())*2, (maxx-minx).cpu().numpy())],
+
+                                [ 0, 0, 0, 1]
+                                ])
+        
+        # render scene
+        r = pyrender.OffscreenRenderer(960, 960)
+
+        color, _ = r.render(scene, flags=RenderFlags.RGBA)
+        # Image.fromarray(color).save(outdir+'/'+name+'_'+str(i)+'.png')
+
+        vid.append(color)
+
+        r.delete()
+
+    out = np.stack(vid, axis=0)
+    imageio.mimwrite(f'output/results.gif', out, duration=50)
+    out_video = mp.VideoFileClip(f'output/results.gif')
+    out_video.write_videofile("output/results.mp4")
+    del out, vertices
+    return f'output/results.mp4'
+
+def predict(clip_text, method='fast'):
+    gc.collect()
+    print('prompt text instruction: {}'.format(clip_text))
+    if torch.cuda.is_available():
+        text = clip.tokenize([clip_text], truncate=True).cuda()
+    else:
+        text = clip.tokenize([clip_text], truncate=True)
+    feat_clip_text = clip_model.encode_text(text).float()
+    index_motion = trans_encoder.sample(feat_clip_text[0:1], False)
+    pred_pose = net.forward_decoder(index_motion)
+    pred_xyz = recover_from_ric((pred_pose*std+mean).float(), 22)
+    output_name = hashlib.md5(clip_text.encode()).hexdigest()
+    if method == 'fast':
+        xyz = pred_xyz.reshape(1, -1, 22, 3)
+        pose_vis = plot_3d.draw_to_batch(xyz.detach().cpu().numpy(), title_batch=None, outname=[f'output/results.gif'])
+        out_video = mp.VideoFileClip("output/results.gif")
+        out_video.write_videofile("output/results.mp4")
+        return f'output/results.mp4'
+    elif method == 'slow':
+        output_path = render(pred_xyz.detach().cpu().numpy().squeeze(axis=0), device_id=0, name=output_name)
+        return output_path
+
+        
+# ---- Gradio Layout -----
+video_out = gr.Video(label="Motion", mirror_webcam=False, interactive=False) 
+demo = gr.Blocks()
+demo.encrypt = False
+
+with demo:
+    gr.Markdown('''
+            <div>
+            <h1 style='text-align: center'>Generating Human Motion from Textual Descriptions (T2M-GPT)</h1>
+            This space uses <a href='https://mael-zys.github.io/T2M-GPT/' target='_blank'><b>T2M-GPT models</b></a> based on Vector Quantised-Variational AutoEncoder (VQ-VAE) and Generative Pre-trained Transformer (GPT) for human motion generation from textural descriptions🤗
+            </div>
+        ''')
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown('''
+            <figure>
+              <img src="https://huggingface.co/vumichien/T2M-GPT/resolve/main/demo_slow1.gif" alt="Demo Slow", width="425", height=480/>
+              <figcaption> a man starts off in an up right position with botg arms extended out by his sides, he then brings his arms down to his body and claps his hands together. after this he wals down amd the the left where he proceeds to sit on a seat
+              </figcaption>
+            </figure>
+            ''')
+        with gr.Column():
+            gr.Markdown('''
+            <figure>
+              <img src="https://huggingface.co/vumichien/T2M-GPT/resolve/main/demo_slow2.gif" alt="Demo Slow 2", width="425", height=480/>
+              <figcaption> a person puts their hands together, leans forwards slightly then swings the arms from right to left
+              </figcaption>
+            </figure>
+            ''')
+        with gr.Column():
+            gr.Markdown('''
+            <figure>
+              <img src="https://huggingface.co/vumichien/T2M-GPT/resolve/main/demo_slow3.gif" alt="Demo Slow 3", width="425", height=480/>
+              <figcaption> a man is practicing the waltz with a partner
+              </figcaption>
+            </figure>
+            ''')
+    with gr.Row():
+        with gr.Column():
+            gr.Markdown('''
+            ### Generate human motion by **T2M-GPT**
+            ##### Step 1. Give prompt text describing human motion
+            ##### Step 2. Choice method to render output (Fast: Sketch skeleton; Slow: SMPL mesh, only work with GPU and running time around 2 mins)
+            ##### Step 3. Generate output and enjoy
+            ''')
+    with gr.Column():
+        with gr.Row():
+            text_prompt = gr.Textbox(label="Text prompt", lines=1, interactive=True)
+            method = gr.Dropdown(["slow", "fast"], label="Method", value="slow")
+        with gr.Row():
+            generate_btn = gr.Button("Generate")
+            generate_btn.click(predict, [text_prompt, method], [video_out], api_name="generate")
+        with gr.Row():  
+            video_out.render()
+    with gr.Row(): 
+        gr.Markdown('''
+            ### You can test by following examples:
+            ''')
+    examples = gr.Examples(
+            examples=[
+                ["a person jogs in place, slowly at first, then increases speed. they then back up and squat down.", "slow"],
+                ["a man steps forward and does a handstand", "slow"],
+                ["a man rises from the ground, walks in a circle and sits back down on the ground", "slow"],
+                ["a man starts off in an up right position with botg arms extended out by his sides, he then brings his arms down to his body and claps his hands together. after this he wals down amd the the left where he proceeds to sit on a seat", "slow"],
+                ["a person puts their hands together, leans forwards slightly then swings the arms from right to left","slow"],
+                ["a man is practicing the waltz with a partner","slow"],
+            ],
+            label="Examples", 
+            inputs=[text_prompt, method],
+            outputs=[video_out],
+            fn=predict,
+            cache_examples=True,
+        )
+
+        
+demo.launch(debug=True)

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/packages.txt

@@ -0,0 +1,4 @@
+libgl1-mesa-dev
+libglu1-mesa-dev
+freeglut3-dev
+mesa-common-dev

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/.coveragerc

@@ -0,0 +1,5 @@
+[report]
+exclude_lines =
+    def __repr__
+    def __str__
+    @abc.abstractmethod

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/.flake8

@@ -0,0 +1,8 @@
+[flake8]
+ignore = E231,W504,F405,F403
+max-line-length = 79
+select = B,C,E,F,W,T4,B9
+exclude = 
+    docs/source/conf.py,
+    __pycache__,
+    examples/*

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/.gitignore

@@ -0,0 +1,106 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+docs/**/generated/**
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+- repo: https://gitlab.com/pycqa/flake8
+  rev: 3.7.1
+  hooks:
+    - id: flake8
+      exclude: ^setup.py

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/.travis.yml

@@ -0,0 +1,43 @@
+language: python
+sudo: required
+dist: xenial
+
+python:
+- '3.6'
+- '3.7'
+
+before_install:
+  # Pre-install osmesa
+  - sudo apt update
+  - sudo wget https://github.com/mmatl/travis_debs/raw/master/xenial/mesa_18.3.3-0.deb
+  - sudo dpkg -i ./mesa_18.3.3-0.deb || true
+  - sudo apt install -f
+  - git clone https://github.com/mmatl/pyopengl.git
+  - cd pyopengl
+  - pip install .
+  - cd ..
+
+install:
+  - pip install .
+    #  - pip install -q pytest pytest-cov coveralls
+  - pip install pytest pytest-cov coveralls
+  - pip install ./pyopengl
+
+script:
+  - PYOPENGL_PLATFORM=osmesa pytest --cov=pyrender tests
+
+after_success:
+- coveralls || true
+
+deploy:
+  provider: pypi
+  skip_existing: true
+  user: mmatl
+  on:
+    tags: true
+    branch: master
+  password:
+    secure: O4WWMbTYb2eVYIO4mMOVa6/xyhX7mPvJpd96cxfNvJdyuqho8VapOhzqsI5kahMB1hFjWWr61yR4+Ru5hoDYf3XA6BQVk8eCY9+0H7qRfvoxex71lahKAqfHLMoE1xNdiVTgl+QN9hYjOnopLod24rx8I8eXfpHu/mfCpuTYGyLlNcDP5St3bXpXLPB5wg8Jo1YRRv6W/7fKoXyuWjewk9cJAS0KrEgnDnSkdwm6Pb+80B2tcbgdGvpGaByw5frndwKiMUMgVUownepDU5POQq2p29wwn9lCvRucULxjEgO+63jdbZRj5fNutLarFa2nISfYnrd72LOyDfbJubwAzzAIsy2JbFORyeHvCgloiuE9oE7a9oOQt/1QHBoIV0seiawMWn55Yp70wQ7HlJs4xSGJWCGa5+9883QRNsvj420atkb3cgO8P+PXwiwTi78Dq7Z/xHqccsU0b8poqBneQoA+pUGgNnF6V7Z8e9RsCcse2gAWSZWuOK3ua+9xCgH7I7MeL3afykr2aJ+yFCoYJMFrUjJeodMX2RbL0q+3FzIPZeGW3WdhTEAL9TSKRcJBSQTskaQlZx/OcpobxS7t3d2S68CCLG9uMTqOTYws55WZ1etalA75sRk9K2MR7ZGjZW3jdtvMViISc/t6Rrjea1GE8ZHGJC6/IeLIWA2c7nc=
+  distributions: sdist bdist_wheel
+notifications:
+  email: false

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Matthew Matl
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/MANIFEST.in

@@ -0,0 +1,5 @@
+# Include the license
+include LICENSE
+include README.rst
+include pyrender/fonts/*
+include pyrender/shaders/*

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/README.md

@@ -0,0 +1,92 @@
+# Pyrender
+
+[![Build Status](https://travis-ci.org/mmatl/pyrender.svg?branch=master)](https://travis-ci.org/mmatl/pyrender)
+[![Documentation Status](https://readthedocs.org/projects/pyrender/badge/?version=latest)](https://pyrender.readthedocs.io/en/latest/?badge=latest)
+[![Coverage Status](https://coveralls.io/repos/github/mmatl/pyrender/badge.svg?branch=master)](https://coveralls.io/github/mmatl/pyrender?branch=master)
+[![PyPI version](https://badge.fury.io/py/pyrender.svg)](https://badge.fury.io/py/pyrender)
+[![Downloads](https://pepy.tech/badge/pyrender)](https://pepy.tech/project/pyrender)
+
+Pyrender is a pure Python (2.7, 3.4, 3.5, 3.6) library for physically-based
+rendering and visualization.
+It is designed to meet the [glTF 2.0 specification from Khronos](https://www.khronos.org/gltf/).
+
+Pyrender is lightweight, easy to install, and simple to use.
+It comes packaged with both an intuitive scene viewer and a headache-free
+offscreen renderer with support for GPU-accelerated rendering on headless
+servers, which makes it perfect for machine learning applications.
+
+Extensive documentation, including a quickstart guide, is provided [here](https://pyrender.readthedocs.io/en/latest/).
+
+For a minimal working example of GPU-accelerated offscreen rendering using EGL,
+check out the [EGL Google CoLab Notebook](https://colab.research.google.com/drive/1pcndwqeY8vker3bLKQNJKr3B-7-SYenE?usp=sharing).
+
+
+<p align="center">
+  <img width="48%" src="https://github.com/mmatl/pyrender/blob/master/docs/source/_static/rotation.gif?raw=true" alt="GIF of Viewer"/>
+  <img width="48%" src="https://github.com/mmatl/pyrender/blob/master/docs/source/_static/damaged_helmet.png?raw=true" alt="Damaged Helmet"/>
+</p>
+
+## Installation
+You can install pyrender directly from pip.
+
+```bash
+pip install pyrender
+```
+
+## Features
+
+Despite being lightweight, pyrender has lots of features, including:
+
+* Simple interoperation with the amazing [trimesh](https://github.com/mikedh/trimesh) project,
+which enables out-of-the-box support for dozens of mesh types, including OBJ,
+STL, DAE, OFF, PLY, and GLB.
+* An easy-to-use scene viewer with support for animation, showing face and vertex
+normals, toggling lighting conditions, and saving images and GIFs.
+* An offscreen rendering module that supports OSMesa and EGL backends.
+* Shadow mapping for directional and spot lights.
+* Metallic-roughness materials for physically-based rendering, including several
+types of texture and normal mapping.
+* Transparency.
+* Depth and color image generation.
+
+## Sample Usage
+
+For sample usage, check out the [quickstart
+guide](https://pyrender.readthedocs.io/en/latest/examples/index.html) or one of
+the Google CoLab Notebooks:
+
+* [EGL Google CoLab Notebook](https://colab.research.google.com/drive/1pcndwqeY8vker3bLKQNJKr3B-7-SYenE?usp=sharing)
+
+## Viewer Keyboard and Mouse Controls
+
+When using the viewer, the basic controls for moving about the scene are as follows:
+
+* To rotate the camera about the center of the scene, hold the left mouse button and drag the cursor.
+* To rotate the camera about its viewing axis, hold `CTRL` left mouse button and drag the cursor.
+* To pan the camera, do one of the following:
+    * Hold `SHIFT`, then hold the left mouse button and drag the cursor.
+    * Hold the middle mouse button and drag the cursor.
+* To zoom the camera in or out, do one of the following:
+    * Scroll the mouse wheel.
+    * Hold the right mouse button and drag the cursor.
+
+The available keyboard commands are as follows:
+
+* `a`: Toggles rotational animation mode.
+* `c`: Toggles backface culling.
+* `f`: Toggles fullscreen mode.
+* `h`: Toggles shadow rendering.
+* `i`: Toggles axis display mode (no axes, world axis, mesh axes, all axes).
+* `l`: Toggles lighting mode (scene lighting, Raymond lighting, or direct lighting).
+* `m`: Toggles face normal visualization.
+* `n`: Toggles vertex normal visualization.
+* `o`: Toggles orthographic camera mode.
+* `q`: Quits the viewer.
+* `r`: Starts recording a GIF, and pressing again stops recording and opens a file dialog.
+* `s`: Opens a file dialog to save the current view as an image.
+* `w`: Toggles wireframe mode (scene default, flip wireframes, all wireframe, or all solid).
+* `z`: Resets the camera to the default view.
+
+As a note, displaying shadows significantly slows down rendering, so if you're
+experiencing low framerates, just kill shadows or reduce the number of lights in
+your scene.

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/docs/Makefile

@@ -0,0 +1,23 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+clean:
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	rm -rf ./source/generated/*
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/docs/source/api/index.rst

@@ -0,0 +1,59 @@
+Pyrender API Documentation
+==========================
+
+Constants
+---------
+.. automodapi:: pyrender.constants
+   :no-inheritance-diagram:
+   :no-main-docstr:
+   :no-heading:
+
+Cameras
+-------
+.. automodapi:: pyrender.camera
+   :no-inheritance-diagram:
+   :no-main-docstr:
+   :no-heading:
+
+Lighting
+--------
+.. automodapi:: pyrender.light
+   :no-inheritance-diagram:
+   :no-main-docstr:
+   :no-heading:
+
+Objects
+-------
+.. automodapi:: pyrender
+   :no-inheritance-diagram:
+   :no-main-docstr:
+   :no-heading:
+   :skip: Camera, DirectionalLight, Light, OffscreenRenderer, Node
+   :skip: OrthographicCamera, PerspectiveCamera, PointLight, RenderFlags
+   :skip: Renderer, Scene, SpotLight, TextAlign, Viewer, GLTF
+
+Scenes
+------
+.. automodapi:: pyrender
+   :no-inheritance-diagram:
+   :no-main-docstr:
+   :no-heading:
+   :skip: Camera, DirectionalLight, Light, OffscreenRenderer
+   :skip: OrthographicCamera, PerspectiveCamera, PointLight, RenderFlags
+   :skip: Renderer, SpotLight, TextAlign, Viewer, Sampler, Texture, Material
+   :skip: MetallicRoughnessMaterial, Primitive, Mesh, GLTF
+
+On-Screen Viewer
+----------------
+.. automodapi:: pyrender.viewer
+   :no-inheritance-diagram:
+   :no-inherited-members:
+   :no-main-docstr:
+   :no-heading:
+
+Off-Screen Rendering
+--------------------
+.. automodapi:: pyrender.offscreen
+   :no-inheritance-diagram:
+   :no-main-docstr:
+   :no-heading:

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/docs/source/conf.py

@@ -0,0 +1,352 @@
+# -*- coding: utf-8 -*-
+#
+# core documentation build configuration file, created by
+# sphinx-quickstart on Sun Oct 16 14:33:48 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+from pyrender import __version__
+from sphinx.domains.python import PythonDomain
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('../../'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.coverage',
+    'sphinx.ext.githubpages',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.napoleon',
+    'sphinx.ext.viewcode',
+    'sphinx_automodapi.automodapi',
+    'sphinx_automodapi.smart_resolver'
+]
+numpydoc_class_members_toctree = False
+automodapi_toctreedirnm = 'generated'
+automodsumm_inherited_members = True
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'pyrender'
+copyright = u'2018, Matthew Matl'
+author = u'Matthew Matl'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = __version__
+# The full version, including alpha/beta/rc tags.
+release = __version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+import sphinx_rtd_theme
+html_theme = 'sphinx_rtd_theme'
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (relative to this directory) to use as a favicon of
+# the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
+#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
+#html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# Now only 'ja' uses this config value
+#html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'coredoc'
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+
+# Latex figure (float) alignment
+#'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'pyrender.tex', u'pyrender Documentation',
+     u'Matthew Matl', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'pyrender', u'pyrender Documentation',
+     [author], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'pyrender', u'pyrender Documentation',
+     author, 'pyrender', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False
+
+intersphinx_mapping = {
+    'python' : ('https://docs.python.org/', None),
+    'pyrender' : ('https://pyrender.readthedocs.io/en/latest/', None),
+}
+
+# Autosummary fix
+autosummary_generate = True
+
+# Try to suppress multiple-definition warnings by always taking the shorter
+# path when two or more paths have the same base module
+
+class MyPythonDomain(PythonDomain):
+
+    def find_obj(self, env, modname, classname, name, type, searchmode=0):
+        """Ensures an object always resolves to the desired module
+        if defined there."""
+        orig_matches = PythonDomain.find_obj(
+            self, env, modname, classname, name, type, searchmode
+        )
+
+        if len(orig_matches) <= 1:
+            return orig_matches
+
+        # If multiple matches, try to take the shortest if all the modules are
+        # the same
+        first_match_name_sp = orig_matches[0][0].split('.')
+        base_name = first_match_name_sp[0]
+        min_len = len(first_match_name_sp)
+        best_match = orig_matches[0]
+
+        for match in orig_matches[1:]:
+            match_name = match[0]
+            match_name_sp = match_name.split('.')
+            match_base = match_name_sp[0]
+
+            # If we have mismatched bases, return them all to trigger warnings
+            if match_base != base_name:
+                return orig_matches
+
+            # Otherwise, check and see if it's shorter
+            if len(match_name_sp) < min_len:
+                min_len = len(match_name_sp)
+                best_match = match
+
+        return (best_match,)
+
+
+def setup(sphinx):
+    """Use MyPythonDomain in place of PythonDomain"""
+    sphinx.override_domain(MyPythonDomain)
+

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/docs/source/examples/cameras.rst

@@ -0,0 +1,26 @@
+.. _camera_guide:
+
+Creating Cameras
+================
+
+Pyrender supports three camera types -- :class:`.PerspectiveCamera` and
+:class:`.IntrinsicsCamera` types,
+which render scenes as a human would see them, and
+:class:`.OrthographicCamera` types, which preserve distances between points.
+
+Creating cameras is easy -- just specify their basic attributes:
+
+>>> pc = pyrender.PerspectiveCamera(yfov=np.pi / 3.0, aspectRatio=1.414)
+>>> oc = pyrender.OrthographicCamera(xmag=1.0, ymag=1.0)
+
+For more information, see the Khronos group's documentation here_:
+
+.. _here: https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#projection-matrices
+
+When you add cameras to the scene, make sure that you're using OpenGL camera
+coordinates to specify their pose. See the illustration below for details.
+Basically, the camera z-axis points away from the scene, the x-axis points
+right in image space, and the y-axis points up in image space.
+
+.. image:: /_static/camera_coords.png
+

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/docs/source/examples/index.rst

@@ -0,0 +1,20 @@
+.. _guide:
+
+User Guide
+==========
+
+This section contains guides on how to use Pyrender to quickly visualize
+your 3D data, including a quickstart guide and more detailed descriptions
+of each part of the rendering pipeline.
+
+
+.. toctree::
+   :maxdepth: 2
+
+   quickstart.rst
+   models.rst
+   lighting.rst
+   cameras.rst
+   scenes.rst
+   offscreen.rst
+   viewer.rst

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/docs/source/examples/lighting.rst

@@ -0,0 +1,21 @@
+.. _lighting_guide:
+
+Creating Lights
+===============
+
+Pyrender supports three types of punctual light:
+
+- :class:`.PointLight`: Point-based light sources, such as light bulbs.
+- :class:`.SpotLight`: A conical light source, like a flashlight.
+- :class:`.DirectionalLight`: A general light that does not attenuate with
+  distance.
+
+Creating lights is easy -- just specify their basic attributes:
+
+>>> pl = pyrender.PointLight(color=[1.0, 1.0, 1.0], intensity=2.0)
+>>> sl = pyrender.SpotLight(color=[1.0, 1.0, 1.0], intensity=2.0,
+...                         innerConeAngle=0.05, outerConeAngle=0.5)
+>>> dl = pyrender.DirectionalLight(color=[1.0, 1.0, 1.0], intensity=2.0)
+
+For more information about how these lighting models are implemented,
+see their class documentation.

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/docs/source/examples/models.rst

@@ -0,0 +1,143 @@
+.. _model_guide:
+
+Loading and Configuring Models
+==============================
+The first step to any rendering application is loading your models.
+Pyrender implements the GLTF 2.0 specification, which means that all
+models are composed of a hierarchy of objects.
+
+At the top level, we have a :class:`.Mesh`. The :class:`.Mesh` is
+basically a wrapper of any number of :class:`.Primitive` types,
+which actually represent geometry that can be drawn to the screen.
+
+Primitives are composed of a variety of parameters, including
+vertex positions, vertex normals, color and texture information,
+and triangle indices if smooth rendering is desired.
+They can implement point clouds, triangular meshes, or lines
+depending on how you configure their data and set their
+:attr:`.Primitive.mode` parameter.
+
+Although you can create primitives yourself if you want to,
+it's probably easier to just use the utility functions provided
+in the :class:`.Mesh` class.
+
+Creating Triangular Meshes
+--------------------------
+
+Simple Construction
+~~~~~~~~~~~~~~~~~~~
+Pyrender allows you to create a :class:`.Mesh` containing a
+triangular mesh model directly from a :class:`~trimesh.base.Trimesh` object
+using the :meth:`.Mesh.from_trimesh` static method.
+
+>>> import trimesh
+>>> import pyrender
+>>> import numpy as np
+>>> tm = trimesh.load('examples/models/fuze.obj')
+>>> m = pyrender.Mesh.from_trimesh(tm)
+>>> m.primitives
+[<pyrender.primitive.Primitive at 0x7fbb0af60e50>]
+
+You can also create a single :class:`.Mesh` from a list of
+:class:`~trimesh.base.Trimesh` objects:
+
+>>> tms = [trimesh.creation.icosahedron(), trimesh.creation.cylinder()]
+>>> m = pyrender.Mesh.from_trimesh(tms)
+[<pyrender.primitive.Primitive at 0x7fbb0c2b74d0>,
+ <pyrender.primitive.Primitive at 0x7fbb0c2b7550>]
+
+Vertex Smoothing
+~~~~~~~~~~~~~~~~
+
+The :meth:`.Mesh.from_trimesh` method has a few additional optional parameters.
+If you want to render the mesh without interpolating face normals, which can
+be useful for meshes that are supposed to be angular (e.g. a cube), you
+can specify ``smooth=False``.
+
+>>> m = pyrender.Mesh.from_trimesh(tm, smooth=False)
+
+Per-Face or Per-Vertex Coloration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you have an untextured trimesh, you can color it in with per-face or
+per-vertex colors:
+
+>>> tm.visual.vertex_colors = np.random.uniform(size=tm.vertices.shape)
+>>> tm.visual.face_colors = np.random.uniform(size=tm.faces.shape)
+>>> m = pyrender.Mesh.from_trimesh(tm)
+
+Instancing
+~~~~~~~~~~
+
+If you want to render many copies of the same mesh at different poses,
+you can statically create a vast array of them in an efficient manner.
+Simply specify the ``poses`` parameter to be a list of ``N`` 4x4 homogenous
+transformation matrics that position the meshes relative to their common
+base frame:
+
+>>> tfs = np.tile(np.eye(4), (3,1,1))
+>>> tfs[1,:3,3] = [0.1, 0.0, 0.0]
+>>> tfs[2,:3,3] = [0.2, 0.0, 0.0]
+>>> tfs
+array([[[1. , 0. , 0. , 0. ],
+        [0. , 1. , 0. , 0. ],
+        [0. , 0. , 1. , 0. ],
+        [0. , 0. , 0. , 1. ]],
+       [[1. , 0. , 0. , 0.1],
+        [0. , 1. , 0. , 0. ],
+        [0. , 0. , 1. , 0. ],
+        [0. , 0. , 0. , 1. ]],
+       [[1. , 0. , 0. , 0.2],
+        [0. , 1. , 0. , 0. ],
+        [0. , 0. , 1. , 0. ],
+        [0. , 0. , 0. , 1. ]]])
+
+>>> m = pyrender.Mesh.from_trimesh(tm, poses=tfs)
+
+Custom Materials
+~~~~~~~~~~~~~~~~
+
+You can also specify a custom material for any triangular mesh you create
+in the ``material`` parameter of :meth:`.Mesh.from_trimesh`.
+The main material supported by Pyrender is the
+:class:`.MetallicRoughnessMaterial`.
+The metallic-roughness model supports rendering highly-realistic objects across
+a wide gamut of materials.
+
+For more information, see the documentation of the
+:class:`.MetallicRoughnessMaterial` constructor or look at the Khronos_
+documentation for more information.
+
+.. _Khronos: https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#materials
+
+Creating Point Clouds
+---------------------
+
+Point Sprites
+~~~~~~~~~~~~~
+Pyrender also allows you to create a :class:`.Mesh` containing a
+point cloud directly from :class:`numpy.ndarray` instances
+using the :meth:`.Mesh.from_points` static method.
+
+Simply provide a list of points and optional per-point colors and normals.
+
+>>> pts = tm.vertices.copy()
+>>> colors = np.random.uniform(size=pts.shape)
+>>> m = pyrender.Mesh.from_points(pts, colors=colors)
+
+Point clouds created in this way will be rendered as square point sprites.
+
+.. image:: /_static/points.png
+
+Point Spheres
+~~~~~~~~~~~~~
+If you have a monochromatic point cloud and would like to render it with
+spheres, you can render it by instancing a spherical trimesh:
+
+>>> sm = trimesh.creation.uv_sphere(radius=0.1)
+>>> sm.visual.vertex_colors = [1.0, 0.0, 0.0]
+>>> tfs = np.tile(np.eye(4), (len(pts), 1, 1))
+>>> tfs[:,:3,3] = pts
+>>> m = pyrender.Mesh.from_trimesh(sm, poses=tfs)
+
+.. image:: /_static/points2.png

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/docs/source/examples/offscreen.rst

@@ -0,0 +1,87 @@
+.. _offscreen_guide:
+
+Offscreen Rendering
+===================
+
+.. note::
+   If you're using a headless server, you'll need to use either EGL (for
+   GPU-accelerated rendering) or OSMesa (for CPU-only software rendering).
+   If you're using OSMesa, be sure that you've installed it properly. See
+   :ref:`osmesa` for details.
+
+Choosing a Backend
+------------------
+
+Once you have a scene set up with its geometry, cameras, and lights,
+you can render it using the :class:`.OffscreenRenderer`. Pyrender supports
+three backends for offscreen rendering:
+
+- Pyglet, the same engine that runs the viewer. This requires an active
+  display manager, so you can't run it on a headless server. This is the
+  default option.
+- OSMesa, a software renderer.
+- EGL, which allows for GPU-accelerated rendering without a display manager.
+
+If you want to use OSMesa or EGL, you need to set the ``PYOPENGL_PLATFORM``
+environment variable before importing pyrender or any other OpenGL library.
+You can do this at the command line:
+
+.. code-block:: bash
+
+   PYOPENGL_PLATFORM=osmesa python render.py
+
+or at the top of your Python script:
+
+.. code-block:: bash
+
+   # Top of main python script
+   import os
+   os.environ['PYOPENGL_PLATFORM'] = 'egl'
+
+The handle for EGL is ``egl``, and the handle for OSMesa is ``osmesa``.
+
+Running the Renderer
+--------------------
+
+Once you've set your environment variable appropriately, create your scene and
+then configure the :class:`.OffscreenRenderer` object with a window width,
+a window height, and a size for point-cloud points:
+
+>>> r = pyrender.OffscreenRenderer(viewport_width=640,
+...                                viewport_height=480,
+...                                point_size=1.0)
+
+Then, just call the :meth:`.OffscreenRenderer.render` function:
+
+>>> color, depth = r.render(scene)
+
+.. image:: /_static/scene.png
+
+This will return a ``(w,h,3)`` channel floating-point color image and
+a ``(w,h)`` floating-point depth image rendered from the scene's main camera.
+
+You can customize the rendering process by using flag options from
+:class:`.RenderFlags` and bitwise or-ing them together. For example,
+the following code renders a color image with an alpha channel
+and enables shadow mapping for all directional lights:
+
+>>> flags = RenderFlags.RGBA | RenderFlags.SHADOWS_DIRECTIONAL
+>>> color, depth = r.render(scene, flags=flags)
+
+Once you're done with the offscreen renderer, you need to close it before you
+can run a different renderer or open the viewer for the same scene:
+
+>>> r.delete()
+
+Google CoLab Examples
+---------------------
+
+For a minimal working example of offscreen rendering using OSMesa,
+see the `OSMesa Google CoLab notebook`_.
+
+.. _OSMesa Google CoLab notebook: https://colab.research.google.com/drive/1Z71mHIc-Sqval92nK290vAsHZRUkCjUx
+
+For a minimal working example of offscreen rendering using EGL,
+see the `EGL Google CoLab notebook`_.
+
+.. _EGL Google CoLab notebook: https://colab.research.google.com/drive/1rTLHk0qxh4dn8KNe-mCnN8HAWdd2_BEh

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/docs/source/index.rst

@@ -0,0 +1,41 @@
+.. core documentation master file, created by
+   sphinx-quickstart on Sun Oct 16 14:33:48 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Pyrender Documentation
+========================
+Pyrender is a pure Python (2.7, 3.4, 3.5, 3.6) library for physically-based
+rendering and visualization.
+It is designed to meet the glTF 2.0 specification_ from Khronos
+
+.. _specification: https://www.khronos.org/gltf/
+
+Pyrender is lightweight, easy to install, and simple to use.
+It comes packaged with both an intuitive scene viewer and a headache-free
+offscreen renderer with support for GPU-accelerated rendering on headless
+servers, which makes it perfect for machine learning applications.
+Check out the :ref:`guide` for a full tutorial, or fork me on
+Github_.
+
+.. _Github: https://github.com/mmatl/pyrender
+
+.. image:: _static/rotation.gif
+
+.. image:: _static/damaged_helmet.png
+
+.. toctree::
+   :maxdepth: 2
+
+   install/index.rst
+   examples/index.rst
+   api/index.rst
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/requirements.txt

@@ -0,0 +1,14 @@
+freetype-py
+imageio
+networkx
+numpy
+Pillow
+pyglet==1.4.0a1
+PyOpenGL
+PyOpenGL_accelerate
+six
+trimesh
+sphinx
+sphinx_rtd_theme
+sphinx-automodapi
+

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/pyrender/setup.py

@@ -0,0 +1,76 @@
+"""
+Setup of pyrender Python codebase.
+
+Author: Matthew Matl
+"""
+import sys
+from setuptools import setup
+
+# load __version__
+exec(open('pyrender/version.py').read())
+
+def get_imageio_dep():
+    if sys.version[0] == "2":
+        return 'imageio<=2.6.1'
+    return 'imageio'
+
+requirements = [
+    'freetype-py',                # For font loading
+    get_imageio_dep(),            # For Image I/O
+    'networkx',                   # For the scene graph
+    'numpy',                      # Numpy
+    'Pillow',                     # For Trimesh texture conversions
+    'pyglet>=1.4.10',             # For the pyglet viewer
+    'PyOpenGL~=3.1.0',            # For OpenGL
+#    'PyOpenGL_accelerate~=3.1.0', # For OpenGL
+    'scipy',                      # Because of trimesh missing dep
+    'six',                        # For Python 2/3 interop
+    'trimesh',                    # For meshes
+]
+
+dev_requirements = [
+    'flake8',            # Code formatting checker
+    'pre-commit',        # Pre-commit hooks
+    'pytest',            # Code testing
+    'pytest-cov',        # Coverage testing
+    'tox',               # Automatic virtualenv testing
+]
+
+docs_requirements = [
+    'sphinx',            # General doc library
+    'sphinx_rtd_theme',  # RTD theme for sphinx
+    'sphinx-automodapi'  # For generating nice tables
+]
+
+
+setup(
+    name = 'pyrender',
+    version=__version__,
+    description='Easy-to-use Python renderer for 3D visualization',
+    long_description='A simple implementation of Physically-Based Rendering '
+                       '(PBR) in Python. Compliant with the glTF 2.0 standard.',
+    author='Matthew Matl',
+    author_email='matthewcmatl@gmail.com',
+    license='MIT License',
+    url = 'https://github.com/mmatl/pyrender',
+    classifiers = [
+        'Development Status :: 4 - Beta',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: POSIX :: Linux',
+        'Operating System :: MacOS :: MacOS X',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Natural Language :: English',
+        'Topic :: Scientific/Engineering'
+    ],
+    keywords = 'rendering graphics opengl 3d visualization pbr gltf',
+    packages = ['pyrender', 'pyrender.platforms'],
+    setup_requires = requirements,
+    install_requires = requirements,
+    extras_require={
+        'dev': dev_requirements,
+        'docs': docs_requirements,
+    },
+    include_package_data=True
+)

c125d6fbf8bda65345b6edda6e04f79faf4d67cd/requirements.txt

@@ -0,0 +1,27 @@
+git+https://github.com/openai/CLIP.git
+numpy==1.23.3
+matplotlib==3.4.3
+matplotlib-inline==0.1.2
+transformers 
+h5py
+smplx
+shapely 
+freetype-py
+imageio
+networkx
+numpy
+Pillow
+pyglet==1.4.0a1
+PyOpenGL==3.1.7
+PyOpenGL_accelerate==3.1.7
+six
+trimesh
+sphinx
+sphinx_rtd_theme
+sphinx-automodapi
+mapbox_earcut 
+chumpy 
+gdown 
+MoviePy
+ffmpeg
+torch

checkpoints/kit/kit/Comp_v6_KLD005/opt.txt

@@ -0,0 +1,54 @@
+------------ Options -------------
+batch_size: 32
+checkpoints_dir: ./checkpoints
+dataset_name: kit
+decomp_name: Decomp_SP001_SM001_H512
+dim_att_vec: 512
+dim_dec_hidden: 1024
+dim_movement2_dec_hidden: 512
+dim_movement_dec_hidden: 512
+dim_movement_enc_hidden: 512
+dim_movement_latent: 512
+dim_msd_hidden: 512
+dim_pos_hidden: 1024
+dim_pri_hidden: 1024
+dim_seq_de_hidden: 512
+dim_seq_en_hidden: 512
+dim_text_hidden: 512
+dim_z: 128
+early_stop_count: 3
+estimator_mod: bigru
+eval_every_e: 5
+feat_bias: 5
+fixed_steps: 5
+gpu_id: 2
+input_z: False
+is_continue: True
+is_train: True
+lambda_fake: 10
+lambda_gan_l: 0.1
+lambda_gan_mt: 0.1
+lambda_gan_mv: 0.1
+lambda_kld: 0.005
+lambda_rec: 1
+lambda_rec_init: 1
+lambda_rec_mot: 1
+lambda_rec_mov: 1
+log_every: 50
+lr: 0.0002
+max_sub_epoch: 50
+max_text_len: 20
+n_layers_dec: 1
+n_layers_msd: 2
+n_layers_pos: 1
+n_layers_pri: 1
+n_layers_seq_de: 2
+n_layers_seq_en: 1
+name: Comp_v6_KLD005
+num_experts: 4
+save_every_e: 10
+save_latest: 500
+text_enc_mod: bigru
+tf_ratio: 0.4
+unit_length: 4
+-------------- End ----------------

checkpoints/kit/kit/text_mot_match/eval/E005.txt

@@ -0,0 +1,4 @@
+Positive Pairs Distance
+3.134 2.099 3.881 3.921 2.285 3.744 2.655 1.721 1.946 2.250 2.924 6.876 3.502 1.861 5.818 3.507 8.488 1.692 1.776 4.583 1.980 2.727 3.955 3.607 5.916 3.186 3.832 1.700 1.664 2.896 3.318 2.128
+Negative Pairs Distance
+4.801 7.064 6.329 6.437 3.465 6.363 8.897 6.655 10.889 6.358 8.022 8.131 3.472 9.457 10.489 3.636 9.595 9.930 12.090 6.514 10.054 2.828 11.206 9.073 6.163 10.645 7.251 7.684 13.491 3.869 8.233 5.459

checkpoints/kit/kit/text_mot_match/eval/E010.txt

@@ -0,0 +1,4 @@
+Positive Pairs Distance
+1.908 2.165 2.165 3.126 1.697 1.459 2.442 1.858 1.139 1.622 2.096 2.478 4.929 3.777 4.985 2.417 1.533 2.688 4.114 2.632 3.061 2.188 5.152 2.518 3.600 4.669 8.488 2.839 3.971 2.343 3.527 3.871
+Negative Pairs Distance
+11.756 2.332 11.629 8.415 1.482 5.168 9.379 10.146 11.044 11.025 11.965 5.885 10.289 5.902 8.258 11.976 8.073 7.024 9.437 4.496 3.879 7.686 4.651 4.576 2.901 14.143 2.428 8.259 5.828 7.087 12.836 13.889

checkpoints/kit/kit/text_mot_match/eval/E015.txt

@@ -0,0 +1,4 @@
+Positive Pairs Distance
+3.850 4.731 5.109 2.265 1.610 2.926 2.168 2.637 2.084 1.510 2.175 2.338 1.857 5.098 1.801 2.207 2.303 1.409 2.606 1.605 2.917 1.752 3.372 2.717 3.066 2.843 2.329 2.640 2.341 4.156 3.331 8.131
+Negative Pairs Distance
+6.084 8.782 9.299 5.081 10.574 12.375 6.629 3.773 12.802 17.637 15.346 12.001 11.877 8.885 5.704 7.574 10.524 11.066 13.124 9.736 2.497 16.383 10.255 14.209 15.128 12.731 10.557 15.367 11.058 11.331 2.683 9.325

checkpoints/kit/kit/text_mot_match/eval/E020.txt

@@ -0,0 +1,4 @@
+Positive Pairs Distance
+1.720 2.590 1.074 3.083 2.166 2.835 6.362 2.816 2.871 1.793 3.421 6.261 1.957 5.514 2.695 2.254 1.668 1.689 4.970 2.880 1.581 4.509 2.255 3.921 2.240 2.384 2.844 2.736 4.322 3.335 3.728 2.677
+Negative Pairs Distance
+11.181 11.564 13.729 7.366 12.419 12.882 8.641 18.567 7.485 7.284 11.086 8.577 5.952 4.970 14.443 13.611 11.813 10.937 13.638 11.140 14.285 8.947 13.830 14.733 11.218 3.280 2.429 11.807 11.222 9.967 10.158 10.779

checkpoints/kit/kit/text_mot_match/eval/E025.txt

@@ -0,0 +1,4 @@
+Positive Pairs Distance
+3.553 1.793 2.145 1.160 1.568 0.923 2.591 7.707 2.747 12.233 5.216 1.305 1.507 2.035 1.736 2.155 1.751 1.618 2.041 2.239 2.825 2.352 3.072 2.020 1.761 5.428 2.252 1.604 2.319 2.266 3.278 3.328
+Negative Pairs Distance
+11.462 3.411 1.404 11.581 3.764 11.409 14.073 4.259 12.997 3.549 7.593 12.030 12.991 7.647 12.337 12.592 14.496 14.496 13.530 13.371 12.775 14.828 11.200 17.627 9.128 9.617 9.297 6.782 14.615 2.304 9.306 10.946

checkpoints/kit/kit/text_mot_match/eval/E030.txt

@@ -0,0 +1,4 @@
+Positive Pairs Distance
+7.378 4.702 10.228 2.154 7.611 0.943 1.770 1.255 5.552 3.161 2.180 2.252 3.278 1.767 1.787 1.450 1.546 2.051 1.476 1.828 1.829 1.423 1.150 2.198 9.395 1.748 2.700 2.304 3.917 4.163 2.811 2.562
+Negative Pairs Distance
+3.884 11.919 5.835 6.898 5.753 19.333 10.732 16.049 10.197 11.546 13.682 12.106 12.993 11.981 13.373 13.464 11.788 11.267 16.719 9.747 4.581 14.785 13.978 9.505 7.465 11.392 11.481 9.757 11.077 16.272 15.701 12.256

checkpoints/kit/kit/text_mot_match/eval/E035.txt

@@ -0,0 +1,4 @@
+Positive Pairs Distance
+2.095 9.020 2.176 1.030 1.653 1.306 1.963 4.513 1.906 1.186 2.731 1.678 1.395 2.232 1.394 4.946 1.057 4.450 3.671 2.931 3.655 1.374 1.361 6.001 5.699 1.579 10.237 2.940 1.622 2.032 7.194 1.069
+Negative Pairs Distance
+13.345 14.638 14.061 19.142 8.843 12.152 15.600 7.368 15.635 13.863 15.644 3.686 13.388 15.807 20.432 11.571 13.876 8.513 13.236 19.330 8.771 14.880 13.005 10.810 12.430 10.102 4.440 18.083 3.135 13.201 5.465 8.700

checkpoints/kit/kit/text_mot_match/eval/E040.txt

@@ -0,0 +1,4 @@
+Positive Pairs Distance
+2.531 1.118 4.384 1.787 1.595 2.184 1.371 1.264 2.086 1.477 1.123 1.240 1.040 11.428 3.469 1.511 1.782 2.528 6.645 1.197 1.967 1.188 3.484 5.019 1.024 1.892 3.185 1.017 2.682 1.116 1.047 1.691
+Negative Pairs Distance
+12.010 18.700 16.704 10.839 15.398 8.070 9.340 14.570 8.946 18.806 4.114 12.699 11.821 4.733 15.875 14.064 17.230 16.277 14.135 22.007 18.454 13.210 8.825 8.937 9.880 16.306 13.555 13.649 5.738 12.752 12.994 1.158

checkpoints/kit/kit/text_mot_match/eval/E045.txt

@@ -0,0 +1,4 @@
+Positive Pairs Distance
+1.412 2.614 2.471 0.842 2.629 7.116 7.275 5.793 1.034 8.645 4.851 2.357 1.155 1.208 1.168 1.698 1.550 1.132 7.423 2.531 1.147 2.240 1.575 1.232 1.455 2.134 1.421 3.723 2.887 1.759 3.806 4.387
+Negative Pairs Distance
+10.150 13.459 14.645 13.875 8.746 12.398 13.303 16.393 22.565 9.265 13.325 9.629 18.867 15.043 18.596 19.413 17.903 8.695 4.948 13.620 14.171 19.699 15.496 11.753 12.914 17.017 7.990 14.401 11.768 8.648 13.926 13.402

checkpoints/kit/kit/text_mot_match/eval/E050.txt

@@ -0,0 +1,4 @@
+Positive Pairs Distance
+5.304 2.162 1.767 7.675 1.446 5.696 2.277 0.717 1.477 12.431 3.673 1.286 2.633 1.883 2.555 1.399 11.572 1.303 3.411 1.521 3.885 0.984 1.210 1.038 5.024 5.886 1.283 1.026 2.056 1.738 3.654 2.659
+Negative Pairs Distance
+17.692 17.993 15.431 10.788 22.819 16.591 23.417 13.203 7.453 17.521 19.102 17.595 7.883 20.475 11.371 17.178 5.380 4.884 17.968 18.298 13.822 19.112 11.842 10.838 14.131 14.906 16.252 14.499 13.878 2.300 11.417 21.558