Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/teaser.png filter=lfs diff=lfs merge=lfs -text
+test.mp4 filter=lfs diff=lfs merge=lfs -text
+test_10.mp4 filter=lfs diff=lfs merge=lfs -text
+test_11.mp4 filter=lfs diff=lfs merge=lfs -text
+test_6.mp4 filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/build/temp.linux-x86_64-3.10/.ninja_deps filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/build/temp.linux-x86_64-3.10/src/droid.o filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/misc/droidcalib.png filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/thirdparty/lietorch/examples/registration/assets/image1.png filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/thirdparty/lietorch/examples/registration/assets/image2.png filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/thirdparty/lietorch/examples/registration/assets/image3.png filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/thirdparty/lietorch/examples/registration/assets/image4.png filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/thirdparty/lietorch/examples/registration/assets/registration.gif filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/thirdparty/lietorch/examples/rgbdslam/assets/floor.png filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/thirdparty/lietorch/examples/rgbdslam/assets/room.png filter=lfs diff=lfs merge=lfs -text
+third_party/DroidCalib/thirdparty/lietorch/lietorch.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,5 @@
+outputs/*
+dataset-generator/
+out/
+__pycache__/
+*.pyc

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "third-party/DROID-SLAM/thirdparty/eigen"]
+	path = third-party/DROID-SLAM/thirdparty/eigen
+	url = https://gitlab.com/libeigen/eigen.git
+[submodule "third_party/GVHMR"]
+	path = third_party/GVHMR
+	url = git@github.com:zju3dv/GVHMR.git

LICENSE

@@ -0,0 +1,36 @@
+NVIDIA License
+
+1. Definitions
+
+“Licensor” means any person or entity that distributes its Work.
+“Work” means (a) the original work of authorship made available under this license, which may include software, documentation, or other files, and (b) any additions to or derivative works  thereof  that are made available under this license.
+The terms “reproduce,” “reproduction,” “derivative works,” and “distribution” have the meaning as provided under U.S. copyright law; provided, however, that for the purposes of this license, derivative works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work.
+Works are “made available” under this license by including in or with the Work either (a) a copyright notice referencing the applicability of this license to the Work, or (b) a copy of this license.
+
+2. License Grant
+
+2.1 Copyright Grant. Subject to the terms and conditions of this license, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free, copyright license to use, reproduce, prepare derivative works of, publicly display, publicly perform, sublicense and distribute its Work and any resulting derivative works in any form.
+
+3. Limitations
+
+3.1 Redistribution. You may reproduce or distribute the Work only if (a) you do so under this license, (b) you include a complete copy of this license with your distribution, and (c) you retain without modification any copyright, patent, trademark, or attribution notices that are present in the Work.
+
+3.2 Derivative Works. You may specify that additional or different terms apply to the use, reproduction, and distribution of your derivative works of the Work (“Your Terms”) only if (a) Your Terms provide that the use limitation in Section 3.3 applies to your derivative works, and (b) you identify the specific derivative works that are subject to Your Terms. Notwithstanding Your Terms, this license (including the redistribution requirements in Section 3.1) will continue to apply to the Work itself.
+
+3.3 Use Limitation. The Work and any derivative works thereof only may be used or intended for use non-commercially. Notwithstanding the foregoing, NVIDIA Corporation and its affiliates may use the Work and any derivative works commercially. As used herein, “non-commercially” means for non-commercial academic purposes only.
+
+3.4 Patent Claims. If you bring or threaten to bring a patent claim against any Licensor (including any claim, cross-claim or counterclaim in a lawsuit) to enforce any patents that you allege are infringed by any Work, then your rights under this license from such Licensor (including the grant in Section 2.1) will terminate immediately.
+
+3.5 Trademarks. This license does not grant any rights to use any Licensor’s or its affiliates’ names, logos, or trademarks, except as necessary to reproduce the notices described in this license.
+
+3.6 Termination. If you violate any term of this license, then your rights under this license (including the grant in Section 2.1) will terminate immediately.
+
+4. Disclaimer of Warranty.
+
+THE WORK IS PROVIDED “AS IS” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF 
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. 
+
+5. Limitation of Liability.
+
+EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+

README.md

@@ -0,0 +1,75 @@
+<p align="center">
+ <h1 align="center"> GEM: A Generalist Model for Human Motion</h1>
+  <p align="center">
+    <a href="https://jeffli.site/"><strong>Jiefeng Li</strong></a>
+    ·
+    <a href="https://www.jinkuncao.com/"><strong>Jinkun Cao</strong></a>
+    ·
+    <a href="https://cs.stanford.edu/~haotianz/"><strong>Haotian Zhang</strong></a>
+    ·
+    <a href="https://davrempe.github.io/"><strong>Davis Rempe</strong></a>
+    ·
+    <a href="https://jankautz.com/"><strong>Jan Kautz</strong></a>
+    ·
+    <a href="https://www.umariqbal.info/"><strong>Umar Iqbal</strong></a>
+    ·
+    <a href="https://ye-yuan.com/"><strong>Ye Yuan</strong></a>
+  </p>
+  <h2 align="center">ICCV 2025 (Highlight)</h2>
+  <div align="center">
+    <img src="./assets/teaser.png" alt="Logo" width="100%">
+  </div>
+</p>
+<p align="center">
+  <a href="https://research.nvidia.com/labs/dair/gem/"><img src="https://img.shields.io/badge/Project-Page-0099cc"></a>
+  <a href="https://arxiv.org/abs/2505.01425"><img src="https://img.shields.io/badge/arXiv-2505.01425-b31b1b.svg"></a>
+
+</p>
+
+**GEM** is a generalist model for human motion that handles multiple tasks with a single model, supporting diverse conditioning signals including video, keypoints, text, audio, and 3D keyframes.
+
+---
+
+## 📰 News
+- **[December 2025]** 📢 GENMO has been renamed to **GEM**.
+- **[October 2025]** 📢 The **GEM** codebase is **released!**  
+  Stay tuned for the pretrained models and evaluation scripts.  
+  Follow the [project page](https://research.nvidia.com/labs/dair/gem/) for updates and announcements.
+
+
+---
+
+
+## 🚀 Highlights
+
+GEM introduces a **unified generative framework** that connects motion estimation and generation through shared objectives.
+
+- **Unified framework:** Reframes motion estimation as *constrained generation*, allowing a single model to perform both tasks.  
+- **Regression × Diffusion synergy:** Combines the accuracy of regression models with the diversity of diffusion-based generation.  
+- **Estimation-guided training:** Trains effectively on in-the-wild datasets using only 2D or textual supervision.  
+- **Multimodal conditioning:** Supports video, text, audio, 2D/3D keyframes, or even time-varying mixed inputs (e.g., video → text → video).  
+- **Arbitrary-length motion:** Generates continuous, coherent sequences of any duration in one diffusion pass.  
+- **State-of-the-art performance:** Achieves leading results on diverse motion estimation and generation benchmarks.
+
+For more details, visit the **[GEM project page →](https://research.nvidia.com/labs/dair/gem/)**
+
+---
+
+### Pretrained Models
+You can download pretrained models from [Google Drive](https://drive.google.com/file/d/1b1E84G7S0h2n5o0RmrcmKOhRKukOjgsJ/view?usp=sharing).
+
+## 📖 Paper & Citation
+
+**Paper:**  
+[GENMO: A GENeralist Model for Human MOtion](https://arxiv.org/abs/2505.01425)  
+*Jiefeng Li, Jinkun Cao, Haotian Zhang, Davis Rempe, Jan Kautz, Umar Iqbal, Ye Yuan*  
+ICCV, 2025
+
+**BibTeX:**
+```bibtex
+@inproceedings{genmo2025,
+  title     = {GENMO: A GENeralist Model for Human MOtion},
+  author    = {Li, Jiefeng and Cao, Jinkun and Zhang, Haotian and Rempe, Davis and Kautz, Jan and Iqbal, Umar and Yuan, Ye},
+  booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)},
+  year      = {2025}
+}

SyntheticRecorder.cs

@@ -0,0 +1,667 @@
+using UnityEngine;
+using System;
+using System.IO;
+using System.Collections;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+using UnityEngine.SceneManagement;
+using System.Diagnostics; // Required for FFmpeg Process
+
+public class SyntheticRecorder : MonoBehaviour
+{
+    // --- CONFIGURATION CLASSES ---
+    [System.Serializable]
+    public class AvatarConfig
+    {
+        public string avatarName = "Avatar";
+        public GameObject avatarObject;
+        [Header("Animation")]
+        public Animator animator;
+        [Header("Retargeting Link")]
+        public HybridPoseCopier retargeter;
+        public List<GameObject> extraMeshes = new List<GameObject>();
+        public float specificPadding = 40f;
+        [Header("Keypoint Markers (COCO-17 order)")]
+        public List<Transform> customMarkers = new List<Transform>();
+    }
+
+    // --- JSON STRUCTURES ---
+    public class SequenceData { public List<FrameData> frames; }
+    public class FrameData { public int i; public float[] p, t, b; public int s; }
+
+    public class OutputMeta
+    {
+        public int frame_index;
+        public string image_path; 
+        public string avatar_name;
+        public int face_id;
+        public int left_hand_id;
+        public int right_hand_id;
+        public float[] bbox;
+        public float[] kpts_2d;
+        public int[] kpts_vis;
+        public float[] bbox_clip;
+        public float[] cam_intrinsics;
+        public float[] cam_pos_world;
+        public float[] cam_rot_world;
+        public float[] pelvis_pos_world;
+        public float[] pelvis_rot_world;
+        public float[] smpl_incam_quat;
+        public float[] smpl_incam_transl;
+        public float[] smpl_root_incam_transl;
+        public float smpl_root_world_scale;
+        public float[] kpts_3d_world;
+        public float[] smplx_pose;
+        public float[] smplx_betas;
+    }
+
+    [Header("Settings")]
+    public string inputFolderPath = "Assets/StreamingAssets";
+    public string outputFolder = "C:/Temp/SyntheticDataset";
+    public bool startRecordingOnPlay = true;
+    public bool showDebugUI = true;
+    [Tooltip("If true, saves depth_xxxxx.png files to check what the occlusion camera sees.")]
+    public bool saveDebugDepthImages = true; 
+
+    [Header("Video Settings")]
+    public string ffmpegPath = "ffmpeg"; 
+    public int frameRate = 30;
+
+    [Header("Compression (Twitch VOD Simulation)")]
+    [Tooltip("Target Bitrate in kbps. 6000 is High Quality 1080p. 2500 is messy 720p.")]
+    public int targetBitrateKbps = 2500; 
+    [Tooltip("GOP (Group of Pictures) size in seconds. Twitch uses 2 seconds.")]
+    public float gopSizeSeconds = 2.0f;
+
+    [Header("Parallel Processing")]
+    public int workerId = 0;
+    public int totalWorkers = 1;
+
+    [Header("Sequence Naming")]
+    public string sequenceName = "";
+    private string _currentInputJsonPath = "";
+
+    [Header("Occlusion Settings")]
+    public float occlusionBias = 0.02f; 
+
+    [Header("References")]
+    public GameObject characterRoot; 
+    public Camera vtuberCamera;
+    public SyntheticCameraDriver cameraDriver;
+
+    [Header("Randomization")]
+    public List<AvatarConfig> avatarList = new List<AvatarConfig>();
+    public List<string> worldSceneNames = new List<string>();
+    public LoadSceneMode worldSceneLoadMode = LoadSceneMode.Additive;
+    public bool setLoadedWorldSceneActive = true;
+    public string worldMainCameraName = "Main Camera";
+    public string spawnPointToken = "SpawnPoint";
+
+    [Header("Animation Indices")]
+    public int faceMaxId = 5;
+    public int handsMaxId = 5;
+    public int minSwitchFrames = 30;
+    public int maxSwitchFrames = 120;
+    public string paramFaceIndex = "FaceIndex";
+    public string paramLeftHandIndex = "LeftHandIndex";
+    public string paramRightHandIndex = "RightHandIndex";
+
+    [Header("BBOX Accuracy")]
+    public bool useBakedSkinnedMeshForBbox = true;
+    public int bakedVertexStride = 8;
+
+    [Header("Calibration")]
+    public float movementScale = 1.0f;
+    public Vector3 translationOffset = new Vector3(0, 0.05f, 0);
+    public Vector3 globalCoordinateCorrection = new Vector3(-90, 180, 0);
+
+    // --- PRIVATE STATE ---
+    private float _activePadding = 40f;
+    private SequenceData _data;
+    private Transform[] _bones;
+    private Transform _pelvisBone;
+    private List<Transform> _activeMarkers = new List<Transform>();
+    private HybridPoseCopier _activeRetargeter;
+    private Transform _activeAvatarRoot = null;
+    private Animator _activeAnimator = null;
+    private string _activeAvatarName = "";
+    private readonly List<Renderer> _activeBboxRenderers = new List<Renderer>();
+    private Mesh _bakeMesh;
+    private readonly List<Vector3> _bakedVerts = new List<Vector3>(8192);
+    private Texture2D _greenTex, _redTex, _occTex;
+    private Rect _cachedBbox = new Rect(0, 0, 0, 0);
+    private bool _cachedHasBbox = false;
+    private int[] _cachedMarkerVis = null;
+    private string _currentlyLoadedWorldScene = "";
+
+    private Shader _autoDepthShader; 
+    private const int JOINT_COUNT = 22;
+    private static readonly string[] BONE_NAMES = {
+        "pelvis", "left_hip", "right_hip", "spine1", "left_knee", "right_knee", "spine2",
+        "left_ankle", "right_ankle", "spine3", "left_foot", "right_foot", "neck", "left_collar",
+        "right_collar", "head", "left_shoulder", "right_shoulder", "left_elbow", "right_elbow",
+        "left_wrist", "right_wrist"
+    };
+
+    void Start()
+    {
+        Screen.SetResolution(1280, 720, FullScreenMode.Windowed);
+        
+        EnsureDepthShaderExists();
+        _autoDepthShader = Shader.Find("Custom/AutoLinearDepth");
+        if (!_autoDepthShader) UnityEngine.Debug.LogError("Could not load the auto-generated depth shader!");
+
+        _greenTex = new Texture2D(1, 1); _greenTex.SetPixel(0, 0, Color.green); _greenTex.Apply();
+        _redTex = new Texture2D(1, 1); _redTex.SetPixel(0, 0, Color.red); _redTex.Apply();
+        _occTex = new Texture2D(1, 1); _occTex.SetPixel(0, 0, new Color(1, 0, 0, 0.5f)); _occTex.Apply();
+
+        _bakeMesh = new Mesh();
+        _bakeMesh.MarkDynamic();
+
+        if (startRecordingOnPlay)
+            StartCoroutine(ProcessBatch());
+    }
+
+    private void EnsureDepthShaderExists()
+    {
+        string path = "Assets/SyntheticDepth.shader";
+        if (File.Exists(path)) return;
+
+        string shaderCode = @"
+Shader ""Custom/AutoLinearDepth""
+{
+    SubShader
+    {
+        Tags { ""RenderType""="""" ""Queue""=""Geometry"" ""ForceNoShadowCasting""=""True"" } 
+        Cull Off 
+        ZWrite On
+        ZTest LEqual
+        Pass
+        {
+            CGPROGRAM
+            #pragma vertex vert
+            #pragma fragment frag
+            #include ""UnityCG.cginc""
+            struct appdata { float4 vertex : POSITION; };
+            struct v2f { float4 pos : SV_POSITION; float depth : TEXCOORD0; };
+            v2f vert (appdata v) { v2f o; o.pos = UnityObjectToClipPos(v.vertex); o.depth = -UnityObjectToViewPos(v.vertex).z; return o; }
+            float4 frag (v2f i) : SV_Target { return float4(i.depth, 0, 0, 1); }
+            ENDCG
+        }
+    }
+}";
+        File.WriteAllText(path, shaderCode);
+        #if UNITY_EDITOR
+        UnityEditor.AssetDatabase.Refresh();
+        #endif
+        UnityEngine.Debug.Log("Created Aggressive AutoLinearDepth shader at " + path);
+    }
+
+    private IEnumerator ProcessBatch()
+    {
+        string fullInputPath = Path.IsPathRooted(inputFolderPath) ? inputFolderPath : Path.Combine(Application.dataPath, "..", inputFolderPath);
+        if (!Directory.Exists(fullInputPath)) { UnityEngine.Debug.LogError("Input folder missing"); yield break; }
+
+        string[] allFiles = Directory.GetFiles(fullInputPath, "*.json");
+        Array.Sort(allFiles);
+
+        List<string> myFiles = new List<string>();
+        int safeTotalWorkers = Mathf.Max(1, totalWorkers);
+
+        for (int i = 0; i < allFiles.Length; i++)
+            if (i % safeTotalWorkers == workerId) myFiles.Add(allFiles[i]);
+
+        foreach (string file in myFiles)
+        {
+            _currentInputJsonPath = file;
+            sequenceName = Path.GetFileNameWithoutExtension(file);
+
+            Resources.UnloadUnusedAssets();
+            System.GC.Collect();
+
+            yield return StartCoroutine(LoadRandomWorldRoutine());
+
+            RandomizeAvatarAndGatherRenderers();
+            FindAndCacheBones();
+            ApplyRandomSpawnPoint(SceneManager.GetActiveScene());
+
+            yield return StartCoroutine(RecordSingleSequence());
+        }
+
+        #if UNITY_EDITOR
+        UnityEditor.EditorApplication.isPlaying = false;
+        #else
+        Application.Quit();
+        #endif
+    }
+
+    private IEnumerator LoadRandomWorldRoutine()
+    {
+        if (worldSceneNames == null || worldSceneNames.Count == 0) yield break;
+
+        if (!string.IsNullOrEmpty(_currentlyLoadedWorldScene) && worldSceneLoadMode == LoadSceneMode.Additive)
+        {
+            AsyncOperation unloadOp = SceneManager.UnloadSceneAsync(_currentlyLoadedWorldScene);
+            while (unloadOp != null && !unloadOp.isDone) yield return null;
+        }
+
+        string chosen = worldSceneNames[UnityEngine.Random.Range(0, worldSceneNames.Count)].Trim();
+        _currentlyLoadedWorldScene = chosen;
+
+        AsyncOperation loadOp = SceneManager.LoadSceneAsync(chosen, worldSceneLoadMode);
+        while (!loadOp.isDone) yield return null;
+        yield return null;
+
+        Scene loaded = SceneManager.GetSceneByName(chosen);
+        if (!loaded.IsValid()) loaded = SceneManager.GetSceneByPath(chosen);
+
+        if (loaded.IsValid() && loaded.isLoaded)
+        {
+            if (setLoadedWorldSceneActive) SceneManager.SetActiveScene(loaded);
+            BindToWorldMainCameraOrLog(loaded);
+        }
+    }
+
+    IEnumerator RecordSingleSequence()
+    {
+        _data = JsonConvert.DeserializeObject<SequenceData>(File.ReadAllText(_currentInputJsonPath));
+
+        if (!Directory.Exists(outputFolder)) Directory.CreateDirectory(outputFolder);
+        string seqImageDir = Path.Combine(outputFolder, "images", sequenceName);
+        if (!Directory.Exists(seqImageDir)) Directory.CreateDirectory(seqImageDir);
+
+        // --- FFmpeg Setup for VOD SIMULATION ---
+        string videoPath = Path.Combine(outputFolder, $"video_{sequenceName}.mp4").Replace("\\", "/");
+        
+        // VOD SIMULATION LOGIC:
+        // 1. -b:v {bitrate}k -> Forces the encoder to target a specific bandwidth
+        // 2. -maxrate {bitrate}k -> Prevents it from spiking quality during high motion (causes artifacts)
+        // 3. -bufsize {bitrate*2}k -> Standard buffer size for streaming
+        // 4. -g {gop} -> Sets Keyframe Interval. Twitch uses 2 seconds fixed.
+        // 5. -preset ultrafast -> Keeps Unity realtime, but relies on bitrate starvation to cause the artifacts
+        
+        int gopFrames = Mathf.RoundToInt(frameRate * gopSizeSeconds);
+        
+        string ffmpegArgs = $"-y -f rawvideo -vcodec rawvideo -pix_fmt rgb24 " + 
+                            $"-s {Screen.width}x{Screen.height} -r {frameRate} -i - " + 
+                            $"-vf vflip " + 
+                            $"-c:v libx264 " + 
+                            $"-pix_fmt yuv420p " + 
+                            $"-preset ultrafast " + 
+                            $"-b:v {targetBitrateKbps}k -maxrate {targetBitrateKbps}k -bufsize {targetBitrateKbps * 2}k " +
+                            $"-g {gopFrames} " + 
+                            $"\"{videoPath}\"";
+
+        ProcessStartInfo psi = new ProcessStartInfo
+        {
+            FileName = ffmpegPath,
+            Arguments = ffmpegArgs,
+            UseShellExecute = false,
+            RedirectStandardInput = true,
+            CreateNoWindow = true
+        };
+
+        Process ffmpegProcess = null;
+        try 
+        {
+            ffmpegProcess = Process.Start(psi);
+        }
+        catch(Exception e)
+        {
+            UnityEngine.Debug.LogError($"Failed to start FFmpeg. Is it in PATH? Error: {e.Message}");
+            yield break;
+        }
+
+        Texture2D screenTex = new Texture2D(Screen.width, Screen.height, TextureFormat.RGB24, false);
+        RenderTexture depthRT = new RenderTexture(Screen.width, Screen.height, 24, RenderTextureFormat.RFloat);
+        Texture2D depthReadTex = new Texture2D(Screen.width, Screen.height, TextureFormat.RFloat, false);
+
+        string jsonlPath = Path.Combine(outputFolder, $"sequence_{sequenceName}.jsonl");
+
+        int framesUntilSwitch = 0;
+        int currentFaceId = 0, currentLeftHandId = 0, currentRightHandId = 0;
+
+        using (var sw = new StreamWriter(jsonlPath, false))
+        {
+            for (int i = 0; i < _data.frames.Count; i++)
+            {
+                if (framesUntilSwitch <= 0)
+                {
+                    framesUntilSwitch = UnityEngine.Random.Range(minSwitchFrames, maxSwitchFrames + 1);
+                    currentFaceId = UnityEngine.Random.Range(0, faceMaxId + 1);
+                    currentLeftHandId = UnityEngine.Random.Range(0, handsMaxId + 1);
+                    currentRightHandId = UnityEngine.Random.Range(0, handsMaxId + 1);
+                    if (_activeAnimator != null)
+                    {
+                        _activeAnimator.SetInteger(paramFaceIndex, currentFaceId);
+                        _activeAnimator.SetInteger(paramLeftHandIndex, currentLeftHandId);
+                        _activeAnimator.SetInteger(paramRightHandIndex, currentRightHandId);
+                    }
+                }
+                framesUntilSwitch--;
+
+                ApplyFrame(_data.frames[i]);
+                if (cameraDriver != null) cameraDriver.OnFrame(i);
+                if (_activeRetargeter != null) _activeRetargeter.ManualUpdatePose();
+                
+                Physics.SyncTransforms();
+
+                vtuberCamera.clearFlags = CameraClearFlags.SolidColor;
+                vtuberCamera.backgroundColor = Color.black; 
+                vtuberCamera.cullingMask = ~0; 
+
+                yield return new WaitForEndOfFrame();
+
+                if (screenTex.width != Screen.width || screenTex.height != Screen.height)
+                    screenTex.Reinitialize(Screen.width, Screen.height);
+                
+                screenTex.ReadPixels(new Rect(0, 0, Screen.width, Screen.height), 0, 0);
+                screenTex.Apply();
+
+                byte[] rawFrame = screenTex.GetRawTextureData();
+                if (ffmpegProcess != null && !ffmpegProcess.HasExited)
+                {
+                    try {
+                        ffmpegProcess.StandardInput.BaseStream.Write(rawFrame, 0, rawFrame.Length);
+                        ffmpegProcess.StandardInput.BaseStream.Flush();
+                    } catch (Exception ex) {
+                         UnityEngine.Debug.LogError("FFmpeg write error: " + ex.Message);
+                    }
+                }
+
+                RenderTexture origRT = vtuberCamera.targetTexture;
+                CameraClearFlags origFlags = vtuberCamera.clearFlags;
+                Color origBG = vtuberCamera.backgroundColor;
+
+                vtuberCamera.targetTexture = depthRT;
+                vtuberCamera.clearFlags = CameraClearFlags.SolidColor; 
+                vtuberCamera.backgroundColor = new Color(1000f, 0, 0, 0); 
+                
+                if (_autoDepthShader != null) vtuberCamera.RenderWithShader(_autoDepthShader, "");
+                else vtuberCamera.Render(); 
+
+                RenderTexture.active = depthRT;
+                if (depthReadTex.width != Screen.width || depthReadTex.height != Screen.height)
+                    depthReadTex.Reinitialize(Screen.width, Screen.height);
+                depthReadTex.ReadPixels(new Rect(0, 0, Screen.width, Screen.height), 0, 0);
+                depthReadTex.Apply();
+
+                if (saveDebugDepthImages)
+                {
+                    Texture2D visualDepth = new Texture2D(Screen.width, Screen.height, TextureFormat.RGB24, false);
+                    Color[] rawPixels = depthReadTex.GetPixels();
+                    Color[] visPixels = new Color[rawPixels.Length];
+                    float displayRange = 3.0f; 
+                    for (int k = 0; k < rawPixels.Length; k++)
+                    {
+                        float d = rawPixels[k].r;
+                        if (d > 999f) visPixels[k] = Color.white;
+                        else
+                        {
+                            float norm = Mathf.Clamp01(d / displayRange);
+                            visPixels[k] = new Color(norm, norm, norm);
+                        }
+                    }
+                    visualDepth.SetPixels(visPixels);
+                    visualDepth.Apply();
+                    string depthFile = $"depth_{i:D5}.png";
+                    File.WriteAllBytes(Path.Combine(seqImageDir, depthFile), visualDepth.EncodeToPNG());
+                    Destroy(visualDepth);
+                }
+
+                vtuberCamera.targetTexture = origRT;
+                vtuberCamera.clearFlags = origFlags;
+                vtuberCamera.backgroundColor = origBG;
+                RenderTexture.active = null;
+
+                ComputeBoundingBoxCached();
+
+                float H = Screen.height; float W = Screen.width;
+                Vector3 camP0_W = vtuberCamera.transform.TransformPoint(new Vector3(0f, 0f, 1f));
+                Vector3 camPx_W = vtuberCamera.transform.TransformPoint(new Vector3(1f, 0f, 1f));
+                Vector3 camPyDown_W = vtuberCamera.transform.TransformPoint(new Vector3(0f, -1f, 1f));
+                Vector3 s0 = vtuberCamera.WorldToScreenPoint(camP0_W);
+                Vector3 sx = vtuberCamera.WorldToScreenPoint(camPx_W);
+                Vector3 sy = vtuberCamera.WorldToScreenPoint(camPyDown_W);
+                float cx = s0.x; float cy = H - s0.y;
+                float fx = sx.x - s0.x; float fy = (H - sy.y) - cy;
+
+                Rect rFull = _cachedHasBbox ? _cachedBbox : new Rect(0, 0, 0, 0);
+                float bbox_x = rFull.x; float bbox_y = H - (rFull.y + rFull.height);
+                float bbox_w = rFull.width; float bbox_h = rFull.height;
+                float clip_x0 = Mathf.Clamp(bbox_x, 0, W);
+                float clip_y0 = Mathf.Clamp(bbox_y, 0, H);
+                float clip_w = Mathf.Max(0, Mathf.Clamp(bbox_x + bbox_w, 0, W) - clip_x0);
+                float clip_h = Mathf.Max(0, Mathf.Clamp(bbox_y + bbox_h, 0, H) - clip_y0);
+
+                var kpts2D = new List<float>();
+                var kptsVis = new List<int>();
+                var kpts3D = new List<float>();
+
+                if (_activeMarkers != null)
+                {
+                    for (int mi = 0; mi < _activeMarkers.Count; mi++)
+                    {
+                        Transform t = _activeMarkers[mi];
+                        if (t == null) { continue; }
+
+                        Vector3 wPos = t.position;
+                        kpts3D.Add(wPos.x); kpts3D.Add(wPos.y); kpts3D.Add(wPos.z);
+                        
+                        Vector3 sPos = vtuberCamera.WorldToScreenPoint(wPos);
+                        float x_px = sPos.x; 
+                        float y_px = H - sPos.y; 
+                        kpts2D.Add(x_px); kpts2D.Add(y_px);
+
+                        int vis = 0; 
+                        if (sPos.z > 0 && x_px >= 0 && x_px < W && sPos.y >= 0 && sPos.y < H)
+                        {
+                            int checkRadius = 2; 
+                            float requiredVisibilityRatio = 0.5f; 
+                            int totalSamples = 0;
+                            int visibleSamples = 0;
+                            float markerDistance = sPos.z;
+
+                            for (int ox = -checkRadius; ox <= checkRadius; ox++)
+                            {
+                                for (int oy = -checkRadius; oy <= checkRadius; oy++)
+                                {
+                                    int px = (int)sPos.x + ox;
+                                    int py = (int)sPos.y + oy;
+                                    if (px >= 0 && px < W && py >= 0 && py < H)
+                                    {
+                                        totalSamples++;
+                                        float pixelDepth = depthReadTex.GetPixel(px, py).r;
+                                        if (pixelDepth >= (markerDistance - occlusionBias))
+                                            visibleSamples++;
+                                    }
+                                }
+                            }
+                            if (totalSamples > 0)
+                            {
+                                float visibilityPct = (float)visibleSamples / totalSamples;
+                                vis = (visibilityPct >= requiredVisibilityRatio) ? 2 : 1;
+                            }
+                            else vis = 1;
+                        }
+                        kptsVis.Add(vis);
+                    }
+                }
+
+                _cachedMarkerVis = kptsVis.ToArray();
+                Transform pelvis = (_bones != null && _bones.Length > 0) ? _bones[0] : null;
+                Quaternion correction = Quaternion.Euler(globalCoordinateCorrection);
+                Quaternion pelvisWorld = (pelvis != null) ? pelvis.rotation : Quaternion.identity;
+                Vector3 pelvisPos = (pelvis != null) ? pelvis.position : Vector3.zero;
+                Quaternion incamQ = Quaternion.Inverse(vtuberCamera.transform.rotation) * (Quaternion.Inverse(correction) * pelvisWorld);
+                Vector3 incamPos = vtuberCamera.transform.InverseTransformPoint(pelvisPos);
+                Vector3 rootIncamPos = (characterRoot != null) ? vtuberCamera.transform.InverseTransformPoint(characterRoot.transform.position) : Vector3.zero;
+
+                var meta = new OutputMeta
+                {
+                    frame_index = i,
+                    image_path = videoPath,
+                    avatar_name = _activeAvatarName,
+                    face_id = currentFaceId,
+                    left_hand_id = currentLeftHandId,
+                    right_hand_id = currentRightHandId,
+                    bbox = new float[] { bbox_x, bbox_y, bbox_w, bbox_h },
+                    bbox_clip = new float[] { clip_x0, clip_y0, clip_w, clip_h },
+                    kpts_2d = kpts2D.ToArray(), kpts_vis = kptsVis.ToArray(),
+                    cam_intrinsics = new float[] { fx, fy, cx, cy },
+                    cam_pos_world = new float[] { vtuberCamera.transform.position.x, vtuberCamera.transform.position.y, vtuberCamera.transform.position.z },
+                    cam_rot_world = new float[] { vtuberCamera.transform.rotation.x, vtuberCamera.transform.rotation.y, vtuberCamera.transform.rotation.z, vtuberCamera.transform.rotation.w },
+                    pelvis_pos_world = new float[] { pelvisPos.x, pelvisPos.y, pelvisPos.z },
+                    pelvis_rot_world = new float[] { pelvisWorld.x, pelvisWorld.y, pelvisWorld.z, pelvisWorld.w },
+                    smpl_incam_quat = new float[] { incamQ.x, incamQ.y, incamQ.z, incamQ.w },
+                    smpl_incam_transl = new float[] { incamPos.x, incamPos.y, incamPos.z },
+                    smpl_root_incam_transl = new float[] { rootIncamPos.x, rootIncamPos.y, rootIncamPos.z },
+                    smpl_root_world_scale = (characterRoot != null) ? characterRoot.transform.lossyScale.x : 1f,
+                    kpts_3d_world = kpts3D.ToArray(),
+                    smplx_pose = _data.frames[i].p, smplx_betas = _data.frames[i].b
+                };
+                sw.WriteLine(JsonConvert.SerializeObject(meta));
+            }
+        }
+
+        if (ffmpegProcess != null && !ffmpegProcess.HasExited)
+        {
+            ffmpegProcess.StandardInput.Close();
+            ffmpegProcess.WaitForExit();
+            ffmpegProcess.Close();
+        }
+        
+        if(screenTex) Destroy(screenTex);
+        if(depthRT) Destroy(depthRT);
+        if(depthReadTex) Destroy(depthReadTex);
+    }
+
+    void ApplyFrame(FrameData f)
+    {
+        if (f.p == null || characterRoot == null) return;
+        Quaternion correction = Quaternion.Euler(globalCoordinateCorrection);
+        characterRoot.transform.localPosition = (correction * (new Vector3(-f.t[0], f.t[1], f.t[2]) * movementScale)) + translationOffset;
+        int floatIdx = 0;
+        for (int i = 0; i < JOINT_COUNT; i++)
+        {
+            if (floatIdx + 2 >= f.p.Length) break;
+            float x = f.p[floatIdx++], y = f.p[floatIdx++], z = f.p[floatIdx++];
+            float angle = Mathf.Sqrt(x * x + y * y + z * z);
+            Quaternion q = Quaternion.identity;
+            if (angle > 1e-6f) { float c = Mathf.Cos(angle * 0.5f), s = Mathf.Sin(angle * 0.5f); q = new Quaternion(-(x / angle) * s, (y / angle) * s, (z / angle) * s, -c); }
+            if (_bones != null && i < _bones.Length && _bones[i] != null) _bones[i].localRotation = (i == 0) ? (correction * q) : q;
+        }
+        if (cameraDriver != null) cameraDriver.SetStyleFromFrameData(f.s);
+    }
+
+    private void FindAndCacheBones()
+    {
+        _bones = new Transform[BONE_NAMES.Length];
+        for (int i = 0; i < BONE_NAMES.Length; i++) _bones[i] = FindDeep(characterRoot.transform, BONE_NAMES[i]);
+        _pelvisBone = (_bones != null && _bones.Length > 0) ? _bones[0] : null;
+    }
+
+    private static Transform FindDeep(Transform root, string name)
+    {
+        if (root.name == name) return root;
+        foreach (Transform child in root) { var res = FindDeep(child, name); if (res) return res; }
+        return null;
+    }
+
+    private void RandomizeAvatarAndGatherRenderers()
+    {
+        if (avatarList == null || avatarList.Count == 0) return;
+        _activeBboxRenderers.Clear(); _activeMarkers.Clear();
+        int randomIndex = UnityEngine.Random.Range(0, avatarList.Count);
+        AvatarConfig selected = avatarList[randomIndex];
+        _activeAvatarName = selected.avatarName;
+        _activeAnimator = selected.animator;
+        if (_activeAnimator == null && selected.avatarObject != null) _activeAnimator = selected.avatarObject.GetComponent<Animator>();
+        for (int i = 0; i < avatarList.Count; i++) if (avatarList[i].avatarObject != null) avatarList[i].avatarObject.SetActive(i == randomIndex);
+        _activePadding = selected.specificPadding; _activeRetargeter = selected.retargeter;
+        if (selected.avatarObject != null) _activeAvatarRoot = selected.avatarObject.transform;
+        if (selected.customMarkers != null) _activeMarkers.AddRange(selected.customMarkers);
+        if (characterRoot != null) { foreach (var r in characterRoot.GetComponentsInChildren<Renderer>(true)) _activeBboxRenderers.Add(r); }
+        foreach (GameObject extra in selected.extraMeshes) if (extra) { foreach (var cr in extra.GetComponentsInChildren<Renderer>(true)) if (!_activeBboxRenderers.Contains(cr)) _activeBboxRenderers.Add(cr); }
+    }
+
+    private void ApplyRandomSpawnPoint(Scene worldScene)
+    {
+        if (!worldScene.IsValid() || !worldScene.isLoaded) return;
+        List<Transform> spawns = new List<Transform>();
+        foreach (GameObject root in worldScene.GetRootGameObjects()) { foreach (Transform child in root.GetComponentsInChildren<Transform>(true)) if (child.name.Contains(spawnPointToken)) spawns.Add(child); }
+        if (spawns.Count > 0)
+        {
+            Transform chosen = spawns[UnityEngine.Random.Range(0, spawns.Count)];
+            this.transform.position = chosen.position; this.transform.rotation = chosen.rotation;
+            if (characterRoot != null) { characterRoot.transform.localPosition = Vector3.zero; characterRoot.transform.localRotation = Quaternion.identity; }
+        }
+    }
+
+    private void BindToWorldMainCameraOrLog(Scene worldScene)
+    {
+        Camera found = null;
+        foreach (GameObject root in worldScene.GetRootGameObjects())
+        {
+            foreach (Transform t in root.GetComponentsInChildren<Transform>(true))
+                if (t.name == worldMainCameraName && t.GetComponent<Camera>()) { found = t.GetComponent<Camera>(); break; }
+            if (found) break;
+        }
+        if (!found) return;
+        Vector3 ls = found.transform.lossyScale;
+        if (Mathf.Abs(ls.x - 1f) > 1e-4f || Mathf.Abs(ls.y - 1f) > 1e-4f || Mathf.Abs(ls.z - 1f) > 1e-4f)
+        {
+            found.transform.SetParent(null, true);
+            found.transform.localScale = Vector3.one;
+        }
+        if (vtuberCamera && vtuberCamera != found) vtuberCamera.enabled = false;
+        vtuberCamera = found;
+        cameraDriver = found.GetComponent<SyntheticCameraDriver>() ?? found.gameObject.AddComponent<SyntheticCameraDriver>();
+        cameraDriver.BindAndInit(found, characterRoot.transform);
+    }
+
+    private bool ComputeBoundingBoxCached()
+    {
+        _cachedHasBbox = false; _cachedBbox = new Rect(0, 0, 0, 0);
+        if (vtuberCamera == null || _activeBboxRenderers.Count == 0) return false;
+        float minVX = float.MaxValue, maxVX = float.MinValue, minVY = float.MaxValue, maxVY = float.MinValue;
+        bool foundAny = false;
+        int stride = Mathf.Max(1, bakedVertexStride);
+        foreach (var rend in _activeBboxRenderers)
+        {
+            if (!rend) continue;
+            if (useBakedSkinnedMeshForBbox && rend is SkinnedMeshRenderer smr)
+            {
+                _bakeMesh.Clear(); smr.BakeMesh(_bakeMesh); _bakedVerts.Clear(); _bakeMesh.GetVertices(_bakedVerts);
+                Matrix4x4 localToWorldNoScale = Matrix4x4.TRS(smr.transform.position, smr.transform.rotation, Vector3.one);
+                for (int vi = 0; vi < _bakedVerts.Count; vi += stride)
+                {
+                    Vector3 vp = vtuberCamera.WorldToViewportPoint(localToWorldNoScale.MultiplyPoint3x4(_bakedVerts[vi]));
+                    if (vp.z <= 0f) continue;
+                    foundAny = true; minVX = Math.Min(minVX, vp.x); maxVX = Math.Max(maxVX, vp.x); minVY = Math.Min(minVY, vp.y); maxVY = Math.Max(maxVY, vp.y);
+                }
+            }
+            else
+            {
+                Bounds b = rend.bounds; Vector3 c = b.center, e = b.extents;
+                Vector3[] corners = { c+new Vector3(-e.x,-e.y,-e.z), c+new Vector3(-e.x,-e.y,e.z), c+new Vector3(-e.x,e.y,-e.z), c+new Vector3(-e.x,e.y,e.z), c+new Vector3(e.x,-e.y,-e.z), c+new Vector3(e.x,-e.y,e.z), c+new Vector3(e.x,e.y,-e.z), c+new Vector3(e.x,e.y,e.z) };
+                foreach (var corner in corners)
+                {
+                    Vector3 vp = vtuberCamera.WorldToViewportPoint(corner);
+                    if (vp.z <= 0f) continue;
+                    foundAny = true; minVX = Math.Min(minVX, vp.x); maxVX = Math.Max(maxVX, vp.x); minVY = Math.Min(minVY, vp.y); maxVY = Math.Max(maxVY, vp.y);
+                }
+            }
+        }
+        if (!foundAny) return false;
+        _cachedBbox = new Rect(minVX * Screen.width - _activePadding, minVY * Screen.height - _activePadding, (maxVX - minVX) * Screen.width + _activePadding * 2, (maxVY - minVY) * Screen.height + _activePadding * 2);
+        _cachedHasBbox = true; return true;
+    }
+
+    void OnGUI()
+    {
+        if (!showDebugUI || vtuberCamera == null) return;
+        if (_cachedHasBbox) { Rect r = _cachedBbox; float invY = Screen.height - (r.y + r.height); GUI.DrawTexture(new Rect(r.x, invY, r.width, 3), _greenTex); GUI.DrawTexture(new Rect(r.x, invY + r.height, r.width, 3), _greenTex); GUI.DrawTexture(new Rect(r.x, invY, 3, r.height), _greenTex); GUI.DrawTexture(new Rect(r.x + r.width, invY, 3, r.height), _greenTex); }
+        if (_activeMarkers != null) { for (int mi = 0; mi < _activeMarkers.Count; mi++) { if (!_activeMarkers[mi]) continue; Vector3 sc = vtuberCamera.WorldToScreenPoint(_activeMarkers[mi].position); if (sc.z > 0) { int vis = (_cachedMarkerVis != null && mi < _cachedMarkerVis.Length) ? _cachedMarkerVis[mi] : 2; GUI.DrawTexture(new Rect(sc.x - 2, Screen.height - sc.y - 2, 4, 4), vis == 1 ? _occTex : _greenTex); } } }
+    }
+}

configs/__init__.py

@@ -0,0 +1,30 @@
+import argparse
+import os
+
+from hydra import compose, initialize_config_module
+from hydra.core.config_store import ConfigStore
+
+os.environ["HYDRA_FULL_ERROR"] = "1"
+
+MainStore = ConfigStore.instance()
+
+
+def parse_args_to_cfg():
+    """
+    Use minimal Hydra API to parse args and return cfg.
+    This function don't do _run_hydra which create log file hierarchy.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config-name", "-cn", default="train")
+    parser.add_argument(
+        "overrides",
+        nargs="*",
+        help="Any key=value arguments to override config values (use dots for.nested=overrides)",
+    )
+    args = parser.parse_args()
+
+    # Cfg
+    with initialize_config_module(version_base="1.3", config_module="configs"):
+        cfg = compose(config_name=args.config_name, overrides=args.overrides)
+
+    return cfg

configs/callbacks/ckpt_saver/every10000s_top100.yaml

@@ -0,0 +1,5 @@
+every10000s_top100:
+  _target_: genmo.callbacks.simple_ckpt_saver.SimpleCkptSaver
+  output_dir: ${output_dir}/checkpoints/
+  every_n_steps: 10000
+  save_top_k: 100

configs/callbacks/lr_monitor/pl.yaml

@@ -0,0 +1,2 @@
+pl:
+  _target_: pytorch_lightning.callbacks.lr_monitor.LearningRateMonitor

configs/callbacks/metric/metric_3dpw.yaml

@@ -0,0 +1,2 @@
+metric_3dpw:
+  _target_: genmo.callbacks.metric.metric_3dpw.MetricMocap

configs/callbacks/metric/metric_3dpw_occ.yaml

@@ -0,0 +1,2 @@
+metric_3dpw_occ:
+  _target_: genmo.callbacks.metric.metric_3dpw_occ.MetricMocap

configs/callbacks/metric/metric_aistpp.yaml

@@ -0,0 +1,2 @@
+metric_aistpp:
+  _target_: genmo.callbacks.metric.metric_aistpp.MetricMusic

configs/callbacks/metric/metric_emdb1.yaml

@@ -0,0 +1,4 @@
+metric_emdb1:
+  _target_: genmo.callbacks.metric.metric_emdb.MetricMocap
+  emdb_split: 1
+  occ: false

configs/callbacks/metric/metric_emdb2.yaml

@@ -0,0 +1,4 @@
+metric_emdb2:
+  _target_: genmo.callbacks.metric.metric_emdb.MetricMocap
+  emdb_split: 2
+  occ: false

configs/callbacks/metric/metric_rich.yaml

@@ -0,0 +1,3 @@
+metric_rich:
+  _target_: genmo.callbacks.metric.metric_rich.MetricMocap
+  occ: false

configs/callbacks/metric/metric_unity.yaml

@@ -0,0 +1,4 @@
+metric_unity:
+  _target_: genmo.callbacks.metric.metric_unity.MetricUnity
+  # Disable the old scenepic HTML viz by default (use `vis/vis_unity_val` instead).
+  vis_every_n_val: 1000000000

configs/callbacks/prog_bar/prog_reporter_ed1.yaml

@@ -0,0 +1,5 @@
+prog_reporter_ed1:
+  _target_: genmo.callbacks.prog_bar.ProgressReporter
+  log_every_percent: 0.1
+  exp_name: ${exp_name}
+  data_name: ${data_name}

configs/callbacks/train_speed_timer/base.yaml

@@ -0,0 +1,3 @@
+base:
+  _target_: genmo.callbacks.train_speed_timer.TrainSpeedTimer
+  N_avg: 5

configs/callbacks/vis/vis_music.yaml

@@ -0,0 +1,2 @@
+vis_music:
+  _target_: genmo.callbacks.vis.vis_music.VisMusic

configs/callbacks/vis/vis_speech.yaml

@@ -0,0 +1,2 @@
+vis_speech:
+  _target_: genmo.callbacks.vis.vis_speech.VisSpeech

configs/callbacks/vis/vis_text.yaml

@@ -0,0 +1,2 @@
+vis_text:
+  _target_: genmo.callbacks.vis.vis_text.VisText

configs/callbacks/vis/vis_unity_val.yaml

@@ -0,0 +1,14 @@
+vis_unity_val:
+  _target_: genmo.callbacks.vis.vis_unity_val.VisUnityVal
+  enabled: false
+  every_n_epochs: 1
+  num_batches: 1
+  num_frames: 30
+  render_incam: true
+  render_global: true
+  use_gt_betas_for_pred: true
+  global_root_relative: false
+  crf: 23
+  save_dir: ${output_dir}/vis
+  pred_color: [176, 100, 244]
+  gt_color: [0, 255, 0]

configs/data/collate_cfg/default.yaml

@@ -0,0 +1,23 @@
+max_motion_frames: ${data.dataset_opts.max_motion_frames}
+default_frame_feature_dim:
+  music_array: [1024]
+  music_embed: [35]
+  music_beats: []
+  audio_array: []
+  use_det_kp: []
+
+default_seq_feature_dim:
+  text_embed: [50, 1024]
+
+default_seq_feature_length_multiplier:
+  audio_array: 600
+
+default_feature_val:
+  caption: ""
+  music_fps: 30
+  audio_fps: 30
+  has_text: False
+  # has_audio: False
+  # has_music: False
+
+default_feature_type: {}

configs/data/mocap/trainX_testY.yaml

@@ -0,0 +1,21 @@
+defaults:
+  - collate_cfg: default
+
+# definition of lightning datamodule (dataset + dataloader)
+_target_: genmo.datamodule.mocap_trainX_testY.DataModule
+
+dataset_opts:
+  train: ${train_datasets}
+  val: ${test_datasets}
+  max_motion_frames: 120
+
+loader_opts:
+  train:
+    batch_size: 128
+    num_workers: 8
+  val:
+    batch_size: 1
+    num_workers: 1
+  encoded_music_dim: ${pipeline.args.encoded_music_dim}
+
+limit_each_trainset: null

configs/demo.yaml

@@ -0,0 +1,85 @@
+defaults:
+  # pytorch-lightning
+  - data: ???
+  - model: ???
+  - /text_encoder@model.model_cfg.text_encoder: t5_3b
+  - callbacks: null
+
+  # system
+  - hydra: default
+
+  # utility groups that changes a lot
+  - pipeline: null
+  - network: null
+  - optimizer: null
+  - scheduler: null
+  - train_datasets: null
+  - test_datasets: null
+  - endecoder: null # normalize/unnormalize data
+  - refiner: null
+
+  # global-override
+  - exp: mixed # set "data, model and callbacks" in yaml
+  - global/task: null # dump/test
+  - global/hsearch: null # hyper-param search
+  - global/debug: null # debug mode
+  - _self_
+
+# ================================ #
+#          global setting          #
+# ================================ #
+
+
+# expirement information
+task: fit # [fit, predict]
+exp_name_base: ???
+exp_name_var: ""
+exp_name: ${exp_name_base}_${exp_name_var}
+data_name: ???
+
+# utilities in the entry file
+# output_dir: "outputs/${data_name}/${exp_name}"
+resume_mode: null
+seed: 42
+
+version: null
+ckpt_dir: outputs/${data_name}/${exp_name}/
+remote_results_path: /lustre/fsw/portfolios/nvr/projects/nvr_torontoai_humanmotionfm/workspaces/motiondiff/motiondiff_results/jiefengl/gvhmr
+ckpt_path: null
+
+###
+# W&B logging removed from this repo; TensorBoard is used by `scripts/train.py`.
+rsync_ckpt: true
+
+
+# ================================ #
+#          global setting          #
+# ================================ #
+
+video_name: ???
+output_root: outputs/demo
+output_dir: "${output_root}/${text1_video_name}"
+preprocess_dir: ${output_dir}/preprocess
+video_path: "${output_dir}/0_input_video.mp4"
+
+# Options
+text1: null
+text1_file: null
+text1_video_path: null
+text1_video_name: null
+text_length: 300
+static_cam: False
+verbose: False
+
+paths:
+  bbx: ${preprocess_dir}/bbx.pt
+  bbx_xyxy_video_overlay: ${preprocess_dir}/bbx_xyxy_video_overlay.mp4
+  vit_features: ${preprocess_dir}/vit_features.pt
+  vimo_pred: ${preprocess_dir}/vimo_pred.pt
+  vitpose: ${preprocess_dir}/vitpose.pt
+  vitpose_video_overlay: ${preprocess_dir}/vitpose_video_overlay.mp4
+  hmr4d_results: ${output_dir}/hmr4d_results.pt
+  incam_video: ${output_dir}/1_incam.mp4
+  global_video: ${output_dir}/2_global.mp4
+  incam_global_horiz_video: ${output_dir}/3_incam_global_horiz.mp4
+  slam: ${preprocess_dir}/camera.npy

configs/diffusion/ddim.yaml

@@ -0,0 +1,8 @@
+sampler: ddim
+train_timestep_respacing: ""
+test_timestep_respacing: "50"
+schedule_sampler_type: uniform
+noise_schedule: cosine
+sigma_small: true
+guidance_param: 1.0
+ddim_eta: 0.0

configs/endecoder/v1_amass_local_bedlam_cam.yaml

@@ -0,0 +1,2 @@
+_target_: genmo.network.endecoder.EnDecoder
+stats_name: MM_V1_AMASS_LOCAL_BEDLAM_CAM

configs/exp/genmo_lg.yaml

@@ -0,0 +1,64 @@
+# @package _global_
+defaults:
+  - /diffusion@model_cfg.diffusion: ddim
+  - override /data: mocap/trainX_testY
+  - override /model: genmo
+  - override /network: diffusion
+  - override /pipeline: dual_mode
+  - override /endecoder: v1_amass_local_bedlam_cam
+  - override /optimizer: adamw_2e-4
+  - override /scheduler: epoch_half_200_350
+  - override /train_datasets:
+      - amass_train_v11
+      - humanml3d_static_train
+      - bedlam_v2
+      - h36m_v1
+      - 3dpw_v1
+      - 3dpw_occ_v1
+      - aistpp_train
+      - beat2_static_train
+  - override /test_datasets:
+      # - aistpp_test
+      - humanml3d_eval
+      - emdb1_fliptest
+      - emdb2_fliptest
+      - rich_test
+      - 3dpw_fliptest
+      - 3dpw_occ_fliptest
+  - override /callbacks:
+      - ckpt_saver/every10000s_top100
+      - prog_bar/prog_reporter_ed1
+      - train_speed_timer/base
+      - lr_monitor/pl
+      - vis/vis_text
+      - metric/metric_emdb1
+      - metric/metric_emdb2
+      - metric/metric_rich
+      - metric/metric_3dpw
+      - metric/metric_3dpw_occ
+      # - metric_aistpp
+  - _self_
+
+exp_name_base: ${hydra:runtime.choices.exp}
+exp_name_var: ""
+exp_name: ${exp_name_base}_${exp_name_var}
+data_name: genmo_mixed
+
+multicond_args: null
+
+pl_trainer:
+  precision: 16-mixed
+  log_every_n_steps: 10
+  gradient_clip_val: 0.5
+  max_epochs: null
+  check_val_every_n_epoch: null
+  val_check_interval: 3000
+  max_steps: 200000
+  devices: 1
+  strategy: ddp_find_unused_parameters_true
+
+logger:
+  _target_: pytorch_lightning.loggers.tensorboard.TensorBoardLogger
+  save_dir: ${output_dir}
+  name: ""
+  version: ""

configs/finetune_unity.yaml

@@ -0,0 +1,95 @@
+# genmo/configs/finetune_unity.yaml
+defaults:
+  - train
+  - override /exp: genmo_lg
+  # Keep only generic callbacks; drop dataset-specific metrics/visualizers.
+  - override /callbacks:
+      - ckpt_saver/every10000s_top100
+      - prog_bar/prog_reporter_ed1
+      - train_speed_timer/base
+      - lr_monitor/pl
+      - metric/metric_unity
+      - vis/vis_unity_val
+  - _self_
+
+# Fix logging path mismatch by forcing filename to be local to the run dir
+hydra:
+  job_logging:
+    handlers:
+      file:
+        filename: train.log
+
+# Define mandatory variables and sync output_dir with Hydra run dir
+data_name: "unity"
+exp_name_base: "finetune"
+# Keep `output_dir` from `configs/train.yaml` to avoid a Hydra/OmegaConf interpolation cycle:
+# `hydra.run.dir` -> `${output_dir}` (configs/hydra/default.yaml) and `output_dir` -> `${hydra:run.dir}` would recurse.
+
+# For tiny Unity sets, save a checkpoint at the end of every epoch.
+callbacks:
+  ckpt_saver:
+    every10000s_top100:
+      every_n_steps: null
+      every_n_epochs: 1
+      save_top_k: 1
+  vis:
+    vis_unity_val:
+      enabled: true
+
+train_datasets:
+  unity:
+    _target_: genmo.datasets.unity_dataset.UnityDataset
+    root: "./third_party/GVHMR/processed_dataset"
+    split: "train"
+    motion_frames: 120
+  # Explicitly disable datasets inherited from `exp=genmo_lg`.
+  amass_train_v11: null
+  humanml3d_static_train: null
+  bedlam_v2: null
+  h36m_v1: null
+  3dpw_v1: null
+  3dpw_occ_v1: null
+  aistpp_train: null
+  beat2_static_train: null
+
+test_datasets:
+  unity_val:
+    _target_: genmo.datasets.unity_dataset.UnityDataset
+    root: "./third_party/GVHMR/processed_dataset"
+    split: "train"
+    motion_frames: 120
+  # Explicitly disable test datasets inherited from `exp=genmo_lg`.
+  humanml3d_eval: null
+  emdb1_fliptest: null
+  emdb2_fliptest: null
+  rich_test: null
+  3dpw_fliptest: null
+  3dpw_occ_fliptest: null
+
+# Fine-tuning Hyperparameters
+solver:
+  optimizer:
+    lr: 5e-6 # VERY IMPORTANT: Low LR to preserve pretrained knowledge
+
+  scheduler:
+    type: "constant" # Keep it simple for fine-tuning
+
+# Lightning Trainer settings
+pl_trainer:
+  max_epochs: 5
+  check_val_every_n_epoch: 1
+  log_every_n_steps: 1
+  precision: 16-mixed # Saves VRAM, faster
+  gradient_clip_val: 1.0
+  val_check_interval: 1.0
+
+# Override the default dataloader settings from `exp=genmo_lg` (it uses batch_size=128
+# and the DataModule uses `drop_last=True`, which yields 0 batches for small Unity sets).
+data:
+  loader_opts:
+    train:
+      batch_size: 1
+      num_workers: 1
+    val:
+      batch_size: 1
+      num_workers: 1

configs/hydra/default.yaml

@@ -0,0 +1,19 @@
+# enable color logging
+defaults:
+  - override hydra_logging: colorlog
+  - override job_logging: colorlog
+
+job_logging:
+  formatters:
+    simple:
+      datefmt: "%m/%d %H:%M:%S"
+      format: "[%(asctime)s][%(levelname)s] %(message)s"
+    colorlog:
+      datefmt: "%m/%d %H:%M:%S"
+      format: "[%(cyan)s%(asctime)s%(reset)s][%(log_color)s%(levelname)s%(reset)s] %(message)s"
+  handlers:
+    file:
+      filename: ${output_dir}/${hydra.job.name}.log
+
+run:
+  dir: ${output_dir}

configs/infer_video.yaml

@@ -0,0 +1,67 @@
+defaults:
+  # pytorch-lightning / hydra wiring (kept for compatibility with `exp=...` configs)
+  - data: ???
+  - model: ???
+  - callbacks: null
+  - hydra: default
+  - pipeline: null
+  - network: null
+  - optimizer: null
+  - scheduler: null
+  - train_datasets: null
+  - test_datasets: null
+  - endecoder: null
+  - refiner: null
+
+  # pick an experiment preset (sets data/model/network/pipeline/etc)
+  - exp: genmo_lg
+  - _self_
+
+# Video -> SMPL-X inference (GENMO/GEM)
+video_path: null
+video_name: null
+
+output_root: outputs/infer_video
+output_dir: ${output_root}/${video_name}
+preprocess_dir: ${output_dir}/preprocess
+
+# Checkpoint
+ckpt_path: null
+
+# Inference options
+static_cam: true
+use_kp2d: true
+postproc: true
+resample_to_30fps: true
+verbose: false
+
+# Rendering
+render_incam: true
+render_global: true
+render_side_by_side: true
+render_crf: 23
+
+# Optional visualization: draw estimated camera axes in the global render.
+draw_camera_axes: false
+# Camera pose convention for `paths.slam` (affects camera-axis visualization only):
+# - auto: choose the one closest to the person root each frame
+# - w2c: interpret trajectory as world->camera
+# - c2w: interpret trajectory as camera->world
+camera_pose_convention: auto
+camera_axis_length: 0.5
+camera_axis_width: 3
+
+paths:
+  input_video: ${output_dir}/0_input_video.mp4
+  video_30fps: ${output_dir}/0_input_video_30fps.mp4
+  bbx: ${preprocess_dir}/bbx.pt
+  vitpose: ${preprocess_dir}/vitpose.pt
+  vit_features: ${preprocess_dir}/vit_features.pt
+  hmr4d_results: ${output_dir}/hmr4d_results.pt
+  incam_video: ${output_dir}/1_incam.mp4
+  global_video: ${output_dir}/2_global.mp4
+  incam_global_horiz_video: ${output_dir}/3_incam_global_horiz.mp4
+
+# Disable external logging by default for a local demo script.
+###
+# W&B logging removed from this repo; TensorBoard is used by `scripts/train.py` for training runs.

configs/model/genmo.yaml

@@ -0,0 +1,45 @@
+_target_: genmo.genmo.GENMO
+
+pipeline: ${pipeline}
+optimizer: ${optimizer}
+scheduler: ${scheduler}
+
+model_cfg:
+  train_modes: ["regression", "diffusion"]
+  noisy_2d_obs: true
+  kp2d_noise_scale: 0.5
+  perframe_condition_exists: true
+  train2d_mask_invis_obs: true
+  mask_occluded_imgfeats: true
+  cond_merge_strategy: "add"
+  use_cond_exists_as_input: true
+  normalize_cam_angvel: true
+
+  diffusion:
+    test_timestep_respacing: "50"
+    guidance_param: 2.5
+
+  text_encoder:
+    load_llm: false
+    llm_version: "t5-3b"
+    max_text_len: 50
+
+  condition_mask:
+    mask_img_prob: 0.5
+    mask_cam_prob: 1.0
+    reuse_regression_mask: false
+    regression_no_img_mask: true
+
+  mask_cfg:
+    drop_prob: 0.75
+    max_num_drops: 3
+    min_drop_nframes: 1
+    max_drop_nframes: 30
+  body_mask_cfg:
+    drop_prob: 0.75
+    joint_drop_prob: 0.25
+    max_num_drops: 3
+    min_drop_nframes: 1
+    max_drop_nframes: 30
+  music_mask_prob: 0.1
+  audio_mask_prob: 0.1

configs/network/diffusion.yaml

@@ -0,0 +1,25 @@
+_target_: genmo.network.genmo_diffusion.GENMODiffusion
+args: ${pipeline.args}
+latent_dim: ${.model_cfg.denoiser.latent_dim}
+cond_merge_strategy: "add"
+music_mask_prob: ${.model_cfg.denoiser.music_mask_prob}
+speech_mask_prob: ${.model_cfg.denoiser.speech_mask_prob}
+encoded_music_dim: ${pipeline.args.encoded_music_dim}
+model_cfg:
+  diffusion: ${model_cfg.diffusion}
+  denoiser:
+    _target_: genmo.network.genmo_denoiser.NetworkEncoderRoPE
+    output_dim: 151
+    xt_dim: ${.output_dim}
+    njoints: ${.xt_dim}
+    text_mask_prob: 0.1
+    music_mask_prob: 0.1
+    speech_mask_prob: 0.1
+    use_text_pos_enc: true
+    text_encoder_cfg:
+      mode: all
+      cross_attn_type: mha
+    latent_dim: 1024
+    num_layers: 16
+    num_heads: 8
+    mlp_ratio: 4

configs/optimizer/adamw_2e-4.yaml

@@ -0,0 +1,2 @@
+_target_: torch.optim.AdamW
+lr: 2e-4

configs/pipeline/dual_mode.yaml

@@ -0,0 +1,37 @@
+_target_: genmo.pipeline.genmo_pipeline.Pipeline
+args_denoiser3d: ${network}
+args:
+  endecoder_opt: ${endecoder}
+  use_regression_outputs_prob: 0.
+  use_cfg_sampler_for_gen: true
+  inpaint_x_start_gt: false
+  regression_only: true
+  encoded_music_dim: 35
+  multicond_args: ${multicond_args}
+  infer_version: 2
+  weights:
+    cr_j3d: 500.
+    transl_c: 1.
+    cr_verts: 500.
+    j2d: 1000.
+    j2d_17: 1000.
+    verts2d: 1000.
+
+    proj_gt_j2d_to_bi01: true
+
+    transl_w: 1.
+    static_conf_bce: 1.
+
+  static_conf:
+    vel_thr: 0.15
+
+  in_attr:
+    - obs
+    - f_cliffcam
+    - f_imgseq
+    - f_cam_angvel
+    - encoded_music
+    - encoded_audio
+  mask_out_attr: [] # ${.in_attr}
+  out_attr:
+    pred_cam: 3

configs/scheduler/epoch_half_200_350.yaml

@@ -0,0 +1,6 @@
+scheduler:
+  _target_: torch.optim.lr_scheduler.MultiStepLR
+  milestones: [200, 350]
+  gamma: 0.5
+interval: epoch
+frequency: 1

configs/test_datasets/3dpw_fliptest.yaml

@@ -0,0 +1,3 @@
+3dpw_fliptest:
+  _target_: genmo.datasets.threedpw.threedpw_motion_test.ThreedpwSmplFullSeqDataset
+  flip_test: true

configs/test_datasets/3dpw_occ_fliptest.yaml

@@ -0,0 +1,3 @@
+3dpw_occ_fliptest:
+  _target_: genmo.datasets.threedpw.threedpw_occ_motion_test.ThreedpwOccSmplFullSeqDataset
+  flip_test: true

configs/test_datasets/emdb1_fliptest.yaml

@@ -0,0 +1,4 @@
+emdb1_fliptest:
+  _target_: genmo.datasets.emdb.emdb_motion_test.EmdbSmplFullSeqDataset
+  split: 1
+  flip_test: true

configs/test_datasets/emdb2_fliptest.yaml

@@ -0,0 +1,4 @@
+emdb2_fliptest:
+  _target_: genmo.datasets.emdb.emdb_motion_test.EmdbSmplFullSeqDataset
+  split: 2
+  flip_test: true

configs/test_datasets/humanml3d_eval.yaml

@@ -0,0 +1,7 @@
+humanml3d_eval:
+  _target_: genmo.datasets.pure_motion.humanml3d.Humanml3dDataset
+  eval_gen_only: true
+  cam_augmentation: v11
+  use_random_subset: true
+  random_subset_size: 2
+  random_subset_seed: 7

configs/test_datasets/rich_test.yaml

@@ -0,0 +1,2 @@
+rich_test:
+  _target_: genmo.datasets.rich.rich_motion_test.RichSmplFullSeqDataset

configs/text_encoder/t5_3b.yaml

@@ -0,0 +1,3 @@
+load_llm: true
+llm_version: "t5-3b"
+max_text_len: 50

configs/train.yaml

@@ -0,0 +1,55 @@
+defaults:
+  - _self_
+  # pytorch-lightning
+  - data: ???
+  - model: ???
+  - callbacks: null
+
+  # system
+  - hydra: default
+
+  # utility groups that changes a lot
+  - pipeline: null
+  - network: null
+  - optimizer: null
+  - scheduler: null
+  - train_datasets: null
+  - test_datasets: null
+  - endecoder: null # normalize/unnormalize data
+  - refiner: null
+
+  # global-override
+  - exp: mixed # set "data, model and callbacks" in yaml
+  - global/task: null # dump/test
+  - global/hsearch: null # hyper-param search
+  - global/debug: null # debug mode
+
+# ================================ #
+#          global setting          #
+# ================================ #
+# expirement information
+task: fit # [fit, predict]
+exp_name_base: ???
+exp_name_var: ""
+exp_name: ${exp_name_base}_${exp_name_var}
+data_name: ???
+num_test_data: 32
+
+# utilities in the entry file
+output_dir: "outputs/${data_name}/${exp_name}"
+ckpt_path: null
+resume_mode: null
+seed: 42
+
+# lightning default settings
+pl_trainer:
+  devices: 1
+  num_sanity_val_steps: 0 # disable sanity check
+  precision: 32
+  inference_mode: False
+
+logger:
+  _target_: pytorch_lightning.loggers.tensorboard.TensorBoardLogger
+  save_dir: ${output_dir}
+  name: ""
+  version: ""

configs/train_datasets/3dpw_occ_v1.yaml

@@ -0,0 +1,2 @@
+3dpw_occ_v1:
+  _target_: genmo.datasets.threedpw.threedpw_occ_motion_train.ThreedpwOccSmplDataset

configs/train_datasets/3dpw_v1.yaml

@@ -0,0 +1,2 @@
+3dpw_v1:
+  _target_: genmo.datasets.threedpw.threedpw_motion_train.ThreedpwSmplDataset

configs/train_datasets/aistpp_train.yaml

@@ -0,0 +1,7 @@
+aistpp_train:
+  _target_: genmo.datasets.aistplusplus.aistplusplus.AISTPlusPlusSmplDataset
+  split: train
+  motion_frames: 120
+  lazy_load: false
+  eval_gen_only: true
+  feat_version: v2

configs/train_datasets/amass_train_v11.yaml

@@ -0,0 +1,9 @@
+amass_train_v11:
+  _target_: genmo.datasets.pure_motion.amass.AmassDataset
+
+  motion_frames: 120
+  l_factor: 1.5
+  skip_moyo: True
+  cam_augmentation: v11
+  random1024: False
+  limit_size: null

configs/train_datasets/beat2_static_train.yaml

@@ -0,0 +1,6 @@
+beat2_static_train:
+  _target_: genmo.datasets.beat2.beat2.BEAT2SmplDataset
+  split: train
+  cam_augmentation: static
+  motion_frames: 120
+  lazy_load: false