Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,47 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0131.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0158.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0043.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0018.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0156.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0126.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0023.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0109.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0096.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0173.png filter=lfs diff=lfs merge=lfs -text
+demo_output/test-f/test_full.mp4 filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0064.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0150.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0166.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0025.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0174.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0108.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0125.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0086.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0106.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0161.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0144.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0146.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0127.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0116.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0065.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0132.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0149.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0115.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0100.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0178.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0164.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0133.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0121.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0138.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0119.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0137.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0107.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0145.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0135.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0112.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0136.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0141.png filter=lfs diff=lfs merge=lfs -text
+debug/testmulti_frames_img/frame_0169.png filter=lfs diff=lfs merge=lfs -text

configs/demo.yaml

@@ -0,0 +1,162 @@
+# NAME: DEMO_COSPEECH_fulltask # Experiment names
+NAME : test
+ACCELERATOR: 'gpu' # Devices optioncal: “cpu”, “gpu”, “tpu”, “ipu”, “hpu”, “mps, “auto”
+NUM_NODES: 1 # Number of GPU nodes for distributed training
+DEVICE: [0,1,2,3,4,5,6,7] # Index of gpus eg. [0] or [0,1,2,3,4,5,6,7]
+# DEVICE: [0,1,2,3] # Index of gpus eg. [0] or [0,1,2,3,4,5,6,7]
+
+Selected_part: separate_rot  # defaut separate_rot
+Representation_type:
+  full_h3d:
+    ori: beat_smplx_joints
+    tar: beat_smplx_full
+    vae_test_dim: 263
+  full_rot:
+    ori: beat_smplx_joints
+    tar: beat_smplx_full
+    vae_test_dim: 330
+  compositional:
+    face:
+      ori: beat_smplx_joints
+      tar: beat_smplx_face
+      vae_test_dim: 106
+    hand:
+      ori: beat_smplx_joints
+      tar: beat_smplx_hand
+      vae_test_dim: 180
+    upper:
+      ori: beat_smplx_joints
+      tar: beat_smplx_upper
+      vae_test_dim: 78
+    lower:
+      ori: beat_smplx_joints
+      tar: beat_smplx_lower
+      vae_test_dim: 54
+  separate_rot:
+    face:
+      ori: beat_smplx_joints
+      tar: beat_smplx_face
+      vae_test_dim: 106
+    hand:
+      ori: beat_smplx_joints
+      tar: beat_smplx_hand
+      vae_test_dim: 180
+    upper:
+      ori: beat_smplx_joints
+      tar: beat_smplx_upper
+      vae_test_dim: 78
+    lower:
+      ori: beat_smplx_joints
+      tar: beat_smplx_lower
+      vae_test_dim: 61
+
+TRAIN:
+  #---------------------------------
+  STAGE: lm_instruct # stage "vae" , "lm_pretrain", "lm_instruct"
+  #---------------------------------
+  NUM_WORKERS: 4 # Number of workers
+  BATCH_SIZE: 24 # Size of batches  12
+  END_EPOCH: 999999 # End epoch
+  PRECISION: 'bf16'
+  FORCE_BF16: False
+  RESUME: '' # Resume training from this path
+  PRETRAINED: '' # Preatrained model path
+  PRETRAINED_VQ: './model_files/pretrained_cpt/emage_vq/vq_emage_speaker_2.ckpt' # Preatrained vq model path
+  # PRETRAINED_VQ : './model_files/pretrained_cpt/lom_vq_ds/lom_vq.ckpt'
+  HUBERT_CHECKPOINT: './model_files/hubert_models/hubert_base_ls960.pt'
+  HUBERT_QUANTIZER: './model_files/hubert_models/hubert_base_ls960_L9_km500.bin'
+  OPTIM:
+    target: AdamW
+    params:
+      lr: 2e-4
+      betas: [0.9, 0.99]
+      weight_decay: 0.0
+
+TEST:
+  # CHECKPOINTS: './model_files/pretrained_cpt/lom_a2m/Instruct_Mixed_A2M_LM.ckpt'
+  # CHECKPOINTS : '/home/xiaoyu/WorkSpace/tmp/language_of_motion/experiments/lom/t5-posttrain-newtex4x-t2m/checkpoints/epoch=94.ckpt'
+  # CHECKPOINTS : '/home/xiaoyu/WorkSpace/tmp/language_of_motion/experiments/lom/Qwen3_1_3B_Posttrain_sp2/checkpoints/epoch=49-v1.ckpt'
+  # CHECKPOINTS : '/home/xiaoyu/WorkSpace/tmp/language_of_motion/experiments/lom/T5_moe_posttrain_sp2/checkpoints/epoch=99-v1.ckpt'
+  CHECKPOINTS : './experiments/t5-posttrain-sp2-realtime-finetune-40/checkpoints/epoch=64.ckpt'
+
+  TEST_LENGTH : 40
+  
+  SPLIT: test
+  BATCH_SIZE: 1 
+  HUBERT_CHECKPOINT: './model_files/hubert_models/hubert_base_ls960.pt'
+  HUBERT_QUANTIZER: './model_files/hubert_models/hubert_base_ls960_L9_km500.bin'
+
+  AUDIO_PATH : '/mnt/afs/Multi_datasets/BEAT2/beat_english_v2.0.0/wave16k/2_scott_0_9_9.wav'
+  OUTPUT : '/mnt/afs/WorkSpace/Mogo_Multi_Realtime/demo_output/test-f'
+  DEMO_NAME : 'test'
+  # MAX_LENGTH_SECONDS : 5
+  MAX_LENGTH_SECONDS : 60
+
+LOSS: 
+  LAMBDA_FEATURE: 1.0
+  LAMBDA_VELOCITY: 0.5
+  LAMBDA_COMMIT: 0.02
+  LAMBDA_CLS: 1.0
+ABLATION:
+  RECONS_LOSS: 'l1_smooth'
+
+  
+METRIC:
+  TYPE: [ ]
+
+model:
+  target: lom.models.lom.Language_Motion
+  params:
+    task: 'a2m'
+    lm: ${lm.lom_speaker2}
+    # lm : ${lm.lom_qwen}
+    # lm : ${lm.lom_t5_moe}
+    # lm : ${lm.lom_qwen_vl}
+    # lm : ${lm.lom_qwen_vl}
+    modality_setup:
+      params:
+        audio_fps: ${lm.lom_speaker2.params.audio_samplerate}
+        audio_down: ${lm.lom_speaker2.params.audio_down_sampling}  ## since the hubert model is trained on 320hz audio, and we found also have another version of hubert model trained on 640 downsampling setup
+        motion_fps: ${lm.lom_speaker2.params.motion_framerate}
+        motion_down: ${lm.lom_speaker2.params.motion_down_sampling}
+    modality_tokenizer:
+      vae_face:
+        target: lom.archs.lom_vq.VQVAEConvZero
+        params:
+          vae_layer: 2
+          code_num: 256
+          codebook_size: 256
+          vae_quantizer_lambda: 1
+          vae_test_dim: ${Representation_type.${Selected_part}.face.vae_test_dim}  #dynamic
+      vae_hand:
+        target: lom.archs.lom_vq.VQVAEConvZero
+        params:
+          vae_layer: 2
+          code_num: 256
+          codebook_size: 256
+          vae_quantizer_lambda: 1
+          vae_test_dim: ${Representation_type.${Selected_part}.hand.vae_test_dim}  #dynamic
+      vae_upper:
+        target: lom.archs.lom_vq.VQVAEConvZero
+        params:
+          vae_layer: 2
+          code_num: 256
+          codebook_size: 256
+          vae_quantizer_lambda: 1
+          vae_test_dim: ${Representation_type.${Selected_part}.upper.vae_test_dim}  #dynamic
+      vae_lower:
+        target: lom.archs.lom_vq.VQVAEConvZero
+        params:
+          vae_layer: 4
+          code_num: 256
+          codebook_size: 256
+          vae_quantizer_lambda: 1
+          vae_test_dim: ${Representation_type.${Selected_part}.lower.vae_test_dim}  #dynamic
+      vae_global:
+        target: lom.archs.lom_vq.VAEConvZero
+        params:
+          vae_layer: 4
+          code_num: 256
+          codebook_size: 256
+          vae_quantizer_lambda: 1
+          vae_test_dim: 61

demo.sh

@@ -0,0 +1 @@
+python demo_rec_realtime.py --cfg configs/demo.yaml

demo_output/test-f/test_full.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:728e473c2001440a96be0c459693745fec46646633d344853a0ecd4809e362e1
+size 2036733

demo_output/test/test.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30a4230c30ff0632aea95ed45fef89e8b055e74b6960ca623bb1d98cef0f4331
+size 211176128

demo_output/test_-1/test.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30a4230c30ff0632aea95ed45fef89e8b055e74b6960ca623bb1d98cef0f4331
+size 211176128

single_reder.sh

@@ -0,0 +1 @@
+./third_party/blender-2.93.18-linux-x64/blender --background --python render_slice.py -- --cfg=./configs/render.yaml --dir=debug --mode=video