Add files using upload-large-folder tool

.gitattributes

@@ -1,35 +1,8 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.flac filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,139 @@
 ---
 license: apache-2.0
+library_name: pytorch
+tags:
+  - omnivae
+  - audio-video
+  - vae
+  - t2av
+  - diffusion
+  - synchronization
+  - multimodal-generation
 ---
+
+# OmniVAE Release Assets
+
+This Hugging Face repository contains the release checkpoints, text encoders,
+evaluation data, and metric weights for the
+[OmniVAE GitHub repository](https://github.com/OpenMOSS/OmniVAE).
+
+The public generation weights in this release are T2AV-only. VAE weights are
+included because they are required by the T2AV pipeline.
+
+The source code is intentionally kept out of this repository. Use this bundle
+with the GitHub code by setting `OMNIVAE_RELEASE_ROOT` to the local download
+path.
+
+## Download
+
+```bash
+pip install -U huggingface_hub
+huggingface-cli download OpenMOSS-Team/OmniVAE \
+  --repo-type model \
+  --local-dir /path/to/omnivae_release \
+  --local-dir-use-symlinks False
+
+git clone https://github.com/OpenMOSS/OmniVAE.git
+cd OmniVAE
+export OMNIVAE_RELEASE_ROOT=/path/to/omnivae_release
+export OPEN_SOURCE_ROOT="${OMNIVAE_RELEASE_ROOT}"
+```
+
+Equivalently:
+
+```bash
+source scripts/setup_release_root.sh /path/to/omnivae_release
+```
+
+## Layout
+
+```text
+omnivae_release/
+  manifest.json
+  models/
+    text_encoder/              frozen Qwen3.5 text encoder used by T2AV inference
+    vae/                       OmniVAE checkpoints
+    dit/t2av/                  T2AV generation checkpoints
+  eval/
+    data/t2av/                 validation manifests and small reference data
+    models/t2av/               metric weights for T2AV evaluation
+```
+
+All config and metadata paths are relative to the repository root, so the code
+can load this directory from any local path.
+
+## Text Encoders
+
+The release includes the frozen Qwen3.5 text encoder directory used by the
+validated T2AV configs:
+
+- `models/text_encoder/Qwen3.5-0.8B-Base`
+
+It is a third-party dependency weight, not an additional OmniVAE checkpoint.
+Keeping it preserves the exact inference/evaluation settings used for the
+released results.
+
+## Checkpoints
+
+### VAE
+
+- `models/vae/video_only/recon`
+- `models/vae/video_only/recon_distill`
+- `models/vae/audio_only/recon`
+- `models/vae/audio_only/recon_distill`
+- `models/vae/audio_video/recon_avclip`
+- `models/vae/audio_video/recon_distill_avclip`
+- `models/vae/audio_only/recon_avclip_ft_decoder`
+- `models/vae/audio_only/recon_distill_avclip_ft_decoder`
+
+For T2AV AVCLIP experiment families, the audio branch uses the
+`*_ft_decoder` checkpoint by default.
+
+### Generation DiT
+
+The public bundle contains T2AV generation packages only:
+
+```text
+t2av: t2av_recon, t2av_recon_distill, t2av_recon_avclip, t2av_recon_distill_avclip
+```
+
+Each package is inference-only and excludes optimizer, scheduler, dataloader,
+random state, and trainer state snapshots.
+
+## Evaluation Assets
+
+- `eval/data/t2av/versebench_minimal`: VerseBench validation manifest used by
+  the release smoke and comparison scripts.
+- `eval/models/t2av`: T2AV metric weights for the bundled `my_eval` pipeline.
+
+### rFVD I3D Weight
+
+The rFVD I3D TorchScript weight is not included in this Hugging Face bundle.
+When it is not already present in the installed OmniVAE package, the rFVD
+loader attempts to download it from the upstream Dropbox URL automatically.
+For offline or reproducible evaluation, download the weight separately and
+provide its path explicitly:
+
+```bash
+export I3D_TORCHSCRIPT_PT=/path/to/i3d_torchscript.pt
+bash vae/scripts/eval/run_video_eval_rfvd_only.sh
+```
+
+## Quick Validation
+
+```bash
+cd /path/to/OmniVAE
+export OMNIVAE_RELEASE_ROOT=/path/to/omnivae_release
+
+# T2AV release validation on VerseBench set3-large.
+bash scripts/release_launchers/run_t2av_release_compare_distributed.sh
+```
+
+See the GitHub repository `README.md` and `docs/` for full installation,
+training, inference, and evaluation commands.
+
+## License
+
+The OmniVAE release assets are provided under the Apache-2.0 license. Some
+metric packages and third-party components retain their original licenses; see
+the corresponding source-code subdirectories in the GitHub repository.

manifest.json

@@ -0,0 +1,25 @@
+{
+  "export_format": "omnivae_t2av_open_source_manifest_v1",
+  "exported_at_utc": "2026-06-08T19:02:51+00:00",
+  "root": ".",
+  "sections": {
+    "vae": {
+      "path": "models/vae",
+      "manifest": "models/vae/manifest.json",
+      "description": "VAE checkpoints required by the released T2AV experiments"
+    },
+    "dit": {
+      "path": "models/dit/t2av",
+      "manifest": "models/dit/manifest.json",
+      "description": "T2AV DiT inference packages only"
+    },
+    "text_encoder": {
+      "path": "models/text_encoder/Qwen3.5-0.8B-Base",
+      "description": "Frozen Qwen3.5 text encoder used by the T2AV configs"
+    },
+    "eval": {
+      "path": "eval",
+      "description": "T2AV validation manifests and metric weights"
+    }
+  }
+}

models/dit/manifest.json

@@ -0,0 +1,46 @@
+{
+  "export_format": "omnivae_t2av_open_source_dit_manifest_v1",
+  "package_format": "omnivae_generation_inference_package_v1",
+  "inference_only": true,
+  "exported_at_utc": "2026-06-08T19:02:51+00:00",
+  "output_root": "models/dit/t2av",
+  "num_packages": 4,
+  "packages": [
+    {
+      "task": "t2av",
+      "name": "t2av_recon",
+      "path": "models/dit/t2av/t2av_recon",
+      "relative_path": "models/dit/t2av/t2av_recon",
+      "source_experiment": "2_t2av_recon_lr2",
+      "release_experiment_family": "recon",
+      "metadata": "models/dit/t2av/t2av_recon/metadata.json"
+    },
+    {
+      "task": "t2av",
+      "name": "t2av_recon_avclip",
+      "path": "models/dit/t2av/t2av_recon_avclip",
+      "relative_path": "models/dit/t2av/t2av_recon_avclip",
+      "source_experiment": "2_t2av_recon_avclip_lr2",
+      "release_experiment_family": "recon_avclip",
+      "metadata": "models/dit/t2av/t2av_recon_avclip/metadata.json"
+    },
+    {
+      "task": "t2av",
+      "name": "t2av_recon_distill",
+      "path": "models/dit/t2av/t2av_recon_distill",
+      "relative_path": "models/dit/t2av/t2av_recon_distill",
+      "source_experiment": "2_t2av_recon_distill_lr2",
+      "release_experiment_family": "recon_distill",
+      "metadata": "models/dit/t2av/t2av_recon_distill/metadata.json"
+    },
+    {
+      "task": "t2av",
+      "name": "t2av_recon_distill_avclip",
+      "path": "models/dit/t2av/t2av_recon_distill_avclip",
+      "relative_path": "models/dit/t2av/t2av_recon_distill_avclip",
+      "source_experiment": "2_t2av_recon_distill_avclip_lr2",
+      "release_experiment_family": "recon_distill_avclip",
+      "metadata": "models/dit/t2av/t2av_recon_distill_avclip/metadata.json"
+    }
+  ]
+}

models/dit/t2av/t2av_recon/bridges/bridge_config.json

@@ -0,0 +1,9 @@
+{
+  "bridge_interval": 1,
+  "use_asymmetric_ati": false,
+  "a2v_window_size": 1,
+  "n_bridges": 18,
+  "dim": 1920,
+  "n_heads": 15,
+  "head_dim": 128
+}

models/dit/t2av/t2av_recon/metadata.json

@@ -0,0 +1,29 @@
+{
+  "export_format": "omnivae_generation_inference_package_v1",
+  "inference_only": true,
+  "task": "t2av",
+  "name": "t2av_recon",
+  "source_experiment": "2_t2av_recon_lr2",
+  "exported_at_utc": "2026-06-07T21:30:31+00:00",
+  "text_encoder_model_name_or_path": "models/text_encoder/Qwen3.5-0.8B-Base",
+  "vae_model_name_or_path": "models/vae/video_only/recon",
+  "required_subdirs": [
+    "transformer_video",
+    "transformer_audio",
+    "bridges",
+    "tokenizer",
+    "scheduler"
+  ],
+  "video_vae_model_name_or_path": "models/vae/video_only/recon",
+  "audio_vae_model_path": "models/vae/audio_only/recon",
+  "joint_av": true,
+  "release_vae_paths_rewritten": true,
+  "release_experiment_family": "recon",
+  "release_path_note": "VAE paths are relative to OMNIVAE_RELEASE_ROOT/models/vae. AVCLIP audio branches use ft_decoder checkpoints.",
+  "source_step": 200000,
+  "release_checkpoint_step": 200000,
+  "release_laion_checkpoint_name": "checkpoint-00200000",
+  "source_experiment_original_name": "2_t2av_recon_lr2",
+  "source_checkpoint_provenance": "internal training checkpoint path omitted; use release_checkpoint_step and this package directory.",
+  "release_package_path": "models/dit/t2av/t2av_recon"
+}

models/dit/t2av/t2av_recon/resolved_config.json

@@ -0,0 +1,337 @@
+{
+  "experiment": {
+    "name": "t2av_recon",
+    "output_dir": "models/dit/t2av/t2av_recon"
+  },
+  "dataset": {
+    "type": "av_paired_jsonl",
+    "root": "omitted_public_release_path",
+    "meta_path": null,
+    "jsonl_index_path": null,
+    "jsonl_path_field": "video_path",
+    "jsonl_prompt_field": "prompt_v2",
+    "jsonl_prompt_index_path": null,
+    "slave_path": "",
+    "base_image_path": "",
+    "split": "train",
+    "image_size": 256,
+    "frame_size": [
+      256,
+      256
+    ],
+    "num_frames": 121,
+    "target_fps": 24.0,
+    "decode_backend": "torchcodec",
+    "return_uint8": true,
+    "center_crop": true,
+    "random_flip": false,
+    "num_workers": 6,
+    "prefetch_factor": 2,
+    "pin_memory": true,
+    "drop_last": true,
+    "max_samples": null,
+    "cache_dir": ".cache/omnivae",
+    "repeat": 1,
+    "recaption_prob": 0.0,
+    "captions_file": null,
+    "prompt_templates": [
+      "a high-quality photo of {label}"
+    ],
+    "use_full_label": false,
+    "metadata_paths": [],
+    "dataset_base_path": "/",
+    "sample_rate": 48000,
+    "num_audio_samples": 242000,
+    "max_num_audio_samples": 1440000,
+    "mono": true,
+    "append_duration_suffix": false,
+    "duration_precision": 1,
+    "path_field": "video_path",
+    "prompt_field": "av_caption",
+    "sources": [
+      {
+        "name": "nonspeech_1000w_g2",
+        "path": "data/train/nonspeech_1000w_group2_9376485.jsonl",
+        "weight": 0.5
+      },
+      {
+        "name": "speech_1000w_g2",
+        "path": "data/train/youtube_speech_10042814.jsonl",
+        "weight": 0.35
+      },
+      {
+        "name": "documentary",
+        "path": "data/train/documentary_2627844.jsonl",
+        "weight": 0.12
+      },
+      {
+        "name": "movies",
+        "path": "data/train/movies_8964574.jsonl",
+        "weight": 0.05
+      },
+      {
+        "name": "chinese_drama",
+        "path": "data/train/chinese_drama_3264550.jsonl",
+        "weight": 0.03
+      }
+    ],
+    "task_prefix_enabled": true,
+    "min_clip_duration_ratio": 0.99
+  },
+  "text_encoder": {
+    "model_name_or_path": "models/text_encoder/Qwen3.5-0.8B-Base",
+    "trust_remote_code": false,
+    "torch_dtype": "bfloat16",
+    "attn_implementation": "sdpa",
+    "disable_qwen3_5_fast_path": true,
+    "cache_enabled": false,
+    "compile_model": true,
+    "max_sequence_length": 512,
+    "local_files_only": true
+  },
+  "scheduler": {
+    "model_name_or_path": "models/dit/t2av/t2av_recon",
+    "subfolder": "scheduler",
+    "local_files_only": true,
+    "use_dynamic_shifting": false
+  },
+  "transformer": {
+    "arch": "zimage",
+    "backbone_name_or_path": "Qwen/Qwen3-VL-2B-Instruct",
+    "backbone_torch_dtype": "bfloat16",
+    "init_from_pretrained_backbone": true,
+    "text_config": null,
+    "trust_remote_code": false,
+    "local_files_only": false,
+    "attn_implementation": "flex_attention",
+    "max_sequence_length": 512,
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 16,
+    "dim": 2048,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 16,
+    "n_kv_heads": 16,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      1024,
+      512,
+      512
+    ],
+    "compile_model": true,
+    "attention_backend": null,
+    "bridge_interval": 1,
+    "bridge_enabled": true,
+    "use_asymmetric_ati": false,
+    "a2v_window_size": 1,
+    "compile_bridges": true,
+    "compile_mode": "default",
+    "compile_fullgraph": false,
+    "compile_dynamic": null
+  },
+  "validation": {
+    "modes": [
+      "audio_only",
+      "video_only",
+      "joint_av"
+    ],
+    "joint_av_prompts": {
+      "name": "versebench",
+      "jsonl_path": "eval/data/t2av/versebench_minimal/versebench_t2av_infer_minimal.jsonl",
+      "text_field": "av_caption",
+      "type_field": "type",
+      "index_field": "index",
+      "task_kind": "t2av",
+      "append_duration_suffix": false,
+      "max_examples": null
+    },
+    "video_only_prompts": {
+      "name": "t2v_inline",
+      "max_examples": null,
+      "prompts": [
+        "A cinematic documentary shot of a three-dimensional topographical map, clouds drifting over islands, somber historical lighting.",
+        "A tense cinematic scene inside a dimly lit historical palace hall, guards rushing through a wooden corridor, dramatic torchlight.",
+        "A close-up cinematic drama shot of a young woman lit by cool blue light, anxious expression, soft early 2000s television style.",
+        "A cinematic wide shot of ocean waves rolling onto a sandy beach at sunset, realistic lighting, gentle camera movement.",
+        "A young woman laughing while chatting with a friend at a sunny outdoor cafe.",
+        "An old man telling a story to a group of children sitting around him in a park.",
+        "A street musician playing guitar and singing on a busy city sidewalk.",
+        "A golden retriever running across a grassy field on a bright afternoon.",
+        "Cherry blossoms gently falling along a quiet path in spring.",
+        "A bowl of steaming noodles on a wooden table in a cozy small restaurant."
+      ]
+    },
+    "audio_only_prompt_sets": [
+      {
+        "name": "basetts_valid",
+        "task_kind": "tts",
+        "jsonl_path": "examples/eval/t2a_tts_basetts_sample.jsonl",
+        "text_field": "text",
+        "type_field": "type",
+        "index_field": "index",
+        "append_duration_suffix": true,
+        "duration_precision": 1,
+        "max_examples": 8
+      },
+      {
+        "name": "tta_general_en",
+        "task_kind": "tta",
+        "jsonl_path": "examples/eval/t2a_tta_general_en.jsonl",
+        "text_field": "text",
+        "type_field": "type",
+        "index_field": "index",
+        "append_duration_suffix": true,
+        "duration_precision": 1,
+        "max_examples": null
+      }
+    ],
+    "num_inference_steps": 50,
+    "video_guidance_scale": 4.0,
+    "audio_guidance_scale": 4.0,
+    "cfg_normalization": false,
+    "max_wandb_samples_per_source": 10,
+    "cfg_modes": [
+      "simple",
+      "dual"
+    ],
+    "video_text_guidance": 4.0,
+    "video_modality_guidance": 4.0,
+    "audio_text_guidance": 4.0,
+    "audio_modality_guidance": 4.0,
+    "video_frame_size": [
+      256,
+      256
+    ],
+    "video_num_frames": 121,
+    "video_fps": 24.0,
+    "audio_duration_seconds": 5.0417
+  },
+  "vae": {
+    "type": "univae",
+    "model_name_or_path": "models/vae/video_only/recon",
+    "torch_dtype": "bfloat16",
+    "local_files_only": true,
+    "wan_chunk_mode": "parallel",
+    "compile_encode": true,
+    "compile_encode_mode": "reduce-overhead",
+    "compile_encode_fullgraph": false,
+    "branch": "video",
+    "use_ema": false
+  },
+  "audio_vae": {
+    "type": "univae",
+    "model_path": "models/vae/audio_only/recon",
+    "hop_length": 960,
+    "latent_channels": 128,
+    "torch_dtype": "float32",
+    "compile_encode": true,
+    "compile_encode_fullgraph": false,
+    "compile_encode_mode": "reduce-overhead",
+    "branch": "audio",
+    "use_ema": false
+  },
+  "transformer_video": {
+    "arch": "zimage",
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 48,
+    "dim": 1920,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 15,
+    "n_kv_heads": 15,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      1024,
+      512,
+      512
+    ],
+    "compile_model": true,
+    "attention_backend": null
+  },
+  "transformer_audio": {
+    "arch": "zimage",
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 128,
+    "dim": 1920,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 15,
+    "n_kv_heads": 15,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      2048,
+      16,
+      16
+    ],
+    "compile_model": true,
+    "attention_backend": null
+  },
+  "train": {
+    "seed": 42,
+    "validation_steps": 5000,
+    "validation_prompts": [
+      "a high-quality photo of a goldfish",
+      "a high-quality photo of a tiger shark",
+      "a high-quality photo of an ostrich",
+      "a high-quality photo of a tabby cat"
+    ],
+    "validation_num_inference_steps": 20,
+    "validation_guidance_scale": 4.0,
+    "validation_cfg_normalization": false,
+    "validation_cfg_truncation": 1.0,
+    "validation_num_frames": null,
+    "validation_frame_size": null,
+    "validation_fps": null,
+    "validation_negative_prompts": null,
+    "shift_v": 5.0,
+    "shift_a": 3.0,
+    "validation_force_steps": [
+      0
+    ]
+  }
+}

models/dit/t2av/t2av_recon/resolved_config.yaml

@@ -0,0 +1,290 @@
+experiment:
+  name: t2av_recon
+  output_dir: models/dit/t2av/t2av_recon
+dataset:
+  type: av_paired_jsonl
+  root: omitted_public_release_path
+  meta_path: null
+  jsonl_index_path: null
+  jsonl_path_field: video_path
+  jsonl_prompt_field: prompt_v2
+  jsonl_prompt_index_path: null
+  slave_path: ''
+  base_image_path: ''
+  split: train
+  image_size: 256
+  frame_size:
+  - 256
+  - 256
+  num_frames: 121
+  target_fps: 24.0
+  decode_backend: torchcodec
+  return_uint8: true
+  center_crop: true
+  random_flip: false
+  num_workers: 6
+  prefetch_factor: 2
+  pin_memory: true
+  drop_last: true
+  max_samples: null
+  cache_dir: .cache/omnivae
+  repeat: 1
+  recaption_prob: 0.0
+  captions_file: null
+  prompt_templates:
+  - a high-quality photo of {label}
+  use_full_label: false
+  metadata_paths: []
+  dataset_base_path: /
+  sample_rate: 48000
+  num_audio_samples: 242000
+  max_num_audio_samples: 1440000
+  mono: true
+  append_duration_suffix: false
+  duration_precision: 1
+  path_field: video_path
+  prompt_field: av_caption
+  sources:
+  - name: nonspeech_1000w_g2
+    path: data/train/nonspeech_1000w_group2_9376485.jsonl
+    weight: 0.5
+  - name: speech_1000w_g2
+    path: data/train/youtube_speech_10042814.jsonl
+    weight: 0.35
+  - name: documentary
+    path: data/train/documentary_2627844.jsonl
+    weight: 0.12
+  - name: movies
+    path: data/train/movies_8964574.jsonl
+    weight: 0.05
+  - name: chinese_drama
+    path: data/train/chinese_drama_3264550.jsonl
+    weight: 0.03
+  task_prefix_enabled: true
+  min_clip_duration_ratio: 0.99
+text_encoder:
+  model_name_or_path: models/text_encoder/Qwen3.5-0.8B-Base
+  trust_remote_code: false
+  torch_dtype: bfloat16
+  attn_implementation: sdpa
+  disable_qwen3_5_fast_path: true
+  cache_enabled: false
+  compile_model: true
+  max_sequence_length: 512
+  local_files_only: true
+scheduler:
+  model_name_or_path: models/dit/t2av/t2av_recon
+  subfolder: scheduler
+  local_files_only: true
+  use_dynamic_shifting: false
+transformer:
+  arch: zimage
+  backbone_name_or_path: Qwen/Qwen3-VL-2B-Instruct
+  backbone_torch_dtype: bfloat16
+  init_from_pretrained_backbone: true
+  text_config: null
+  trust_remote_code: false
+  local_files_only: false
+  attn_implementation: flex_attention
+  max_sequence_length: 512
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 16
+  dim: 2048
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 16
+  n_kv_heads: 16
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 1024
+  - 512
+  - 512
+  compile_model: true
+  attention_backend: null
+  bridge_interval: 1
+  bridge_enabled: true
+  use_asymmetric_ati: false
+  a2v_window_size: 1
+  compile_bridges: true
+  compile_mode: default
+  compile_fullgraph: false
+  compile_dynamic: null
+validation:
+  modes:
+  - audio_only
+  - video_only
+  - joint_av
+  joint_av_prompts:
+    name: versebench
+    jsonl_path: eval/data/t2av/versebench_minimal/versebench_t2av_infer_minimal.jsonl
+    text_field: av_caption
+    type_field: type
+    index_field: index
+    task_kind: t2av
+    append_duration_suffix: false
+    max_examples: null
+  video_only_prompts:
+    name: t2v_inline
+    max_examples: null
+    prompts:
+    - A cinematic documentary shot of a three-dimensional topographical map, clouds
+      drifting over islands, somber historical lighting.
+    - A tense cinematic scene inside a dimly lit historical palace hall, guards rushing
+      through a wooden corridor, dramatic torchlight.
+    - A close-up cinematic drama shot of a young woman lit by cool blue light, anxious
+      expression, soft early 2000s television style.
+    - A cinematic wide shot of ocean waves rolling onto a sandy beach at sunset, realistic
+      lighting, gentle camera movement.
+    - A young woman laughing while chatting with a friend at a sunny outdoor cafe.
+    - An old man telling a story to a group of children sitting around him in a park.
+    - A street musician playing guitar and singing on a busy city sidewalk.
+    - A golden retriever running across a grassy field on a bright afternoon.
+    - Cherry blossoms gently falling along a quiet path in spring.
+    - A bowl of steaming noodles on a wooden table in a cozy small restaurant.
+  audio_only_prompt_sets:
+  - name: basetts_valid
+    task_kind: tts
+    jsonl_path: examples/eval/t2a_tts_basetts_sample.jsonl
+    text_field: text
+    type_field: type
+    index_field: index
+    append_duration_suffix: true
+    duration_precision: 1
+    max_examples: 8
+  - name: tta_general_en
+    task_kind: tta
+    jsonl_path: examples/eval/t2a_tta_general_en.jsonl
+    text_field: text
+    type_field: type
+    index_field: index
+    append_duration_suffix: true
+    duration_precision: 1
+    max_examples: null
+  num_inference_steps: 50
+  video_guidance_scale: 4.0
+  audio_guidance_scale: 4.0
+  cfg_normalization: false
+  max_wandb_samples_per_source: 10
+  cfg_modes:
+  - simple
+  - dual
+  video_text_guidance: 4.0
+  video_modality_guidance: 4.0
+  audio_text_guidance: 4.0
+  audio_modality_guidance: 4.0
+  video_frame_size:
+  - 256
+  - 256
+  video_num_frames: 121
+  video_fps: 24.0
+  audio_duration_seconds: 5.0417
+vae:
+  type: univae
+  model_name_or_path: models/vae/video_only/recon
+  torch_dtype: bfloat16
+  local_files_only: true
+  wan_chunk_mode: parallel
+  compile_encode: true
+  compile_encode_mode: reduce-overhead
+  compile_encode_fullgraph: false
+  branch: video
+  use_ema: false
+audio_vae:
+  type: univae
+  model_path: models/vae/audio_only/recon
+  hop_length: 960
+  latent_channels: 128
+  torch_dtype: float32
+  compile_encode: true
+  compile_encode_fullgraph: false
+  compile_encode_mode: reduce-overhead
+  branch: audio
+  use_ema: false
+transformer_video:
+  arch: zimage
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 48
+  dim: 1920
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 15
+  n_kv_heads: 15
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 1024
+  - 512
+  - 512
+  compile_model: true
+  attention_backend: null
+transformer_audio:
+  arch: zimage
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 128
+  dim: 1920
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 15
+  n_kv_heads: 15
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 2048
+  - 16
+  - 16
+  compile_model: true
+  attention_backend: null
+train:
+  seed: 42
+  validation_steps: 5000
+  validation_prompts:
+  - a high-quality photo of a goldfish
+  - a high-quality photo of a tiger shark
+  - a high-quality photo of an ostrich
+  - a high-quality photo of a tabby cat
+  validation_num_inference_steps: 20
+  validation_guidance_scale: 4.0
+  validation_cfg_normalization: false
+  validation_cfg_truncation: 1.0
+  validation_num_frames: null
+  validation_frame_size: null
+  validation_fps: null
+  validation_negative_prompts: null
+  shift_v: 5.0
+  shift_a: 3.0
+  validation_force_steps:
+  - 0

models/dit/t2av/t2av_recon/scheduler/scheduler_config.json

@@ -0,0 +1,18 @@
+{
+  "_class_name": "FlowMatchEulerDiscreteScheduler",
+  "_diffusers_version": "0.37.1",
+  "base_image_seq_len": 256,
+  "base_shift": 0.5,
+  "invert_sigmas": false,
+  "max_image_seq_len": 4096,
+  "max_shift": 1.15,
+  "num_train_timesteps": 1000,
+  "shift": 3.0,
+  "shift_terminal": null,
+  "stochastic_sampling": false,
+  "time_shift_type": "exponential",
+  "use_beta_sigmas": false,
+  "use_dynamic_shifting": false,
+  "use_exponential_sigmas": false,
+  "use_karras_sigmas": false
+}

models/dit/t2av/t2av_recon/tokenizer/chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is true %}
+        {{- '<think>\n' }}
+    {%- else %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

models/dit/t2av/t2av_recon/tokenizer/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": false,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "split_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

models/dit/t2av/t2av_recon/transformer_audio/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "ZImageTransformer2DModel",
+  "_diffusers_version": "0.37.1",
+  "all_f_patch_size": [
+    1
+  ],
+  "all_patch_size": [
+    1
+  ],
+  "axes_dims": [
+    16,
+    56,
+    56
+  ],
+  "axes_lens": [
+    2048,
+    16,
+    16
+  ],
+  "cap_feat_dim": 1024,
+  "dim": 1920,
+  "in_channels": 128,
+  "n_heads": 15,
+  "n_kv_heads": 15,
+  "n_layers": 18,
+  "n_refiner_layers": 2,
+  "norm_eps": 1e-05,
+  "qk_norm": true,
+  "rope_theta": 256.0,
+  "siglip_feat_dim": null,
+  "t_scale": 1000.0
+}

models/dit/t2av/t2av_recon/transformer_video/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "ZImageTransformer2DModel",
+  "_diffusers_version": "0.37.1",
+  "all_f_patch_size": [
+    1
+  ],
+  "all_patch_size": [
+    1
+  ],
+  "axes_dims": [
+    16,
+    56,
+    56
+  ],
+  "axes_lens": [
+    1024,
+    512,
+    512
+  ],
+  "cap_feat_dim": 1024,
+  "dim": 1920,
+  "in_channels": 48,
+  "n_heads": 15,
+  "n_kv_heads": 15,
+  "n_layers": 18,
+  "n_refiner_layers": 2,
+  "norm_eps": 1e-05,
+  "qk_norm": true,
+  "rope_theta": 256.0,
+  "siglip_feat_dim": null,
+  "t_scale": 1000.0
+}

models/dit/t2av/t2av_recon_avclip/resolved_config.json

@@ -0,0 +1,337 @@
+{
+  "experiment": {
+    "name": "t2av_recon_avclip",
+    "output_dir": "models/dit/t2av/t2av_recon_avclip"
+  },
+  "dataset": {
+    "type": "av_paired_jsonl",
+    "root": "omitted_public_release_path",
+    "meta_path": null,
+    "jsonl_index_path": null,
+    "jsonl_path_field": "video_path",
+    "jsonl_prompt_field": "prompt_v2",
+    "jsonl_prompt_index_path": null,
+    "slave_path": "",
+    "base_image_path": "",
+    "split": "train",
+    "image_size": 256,
+    "frame_size": [
+      256,
+      256
+    ],
+    "num_frames": 121,
+    "target_fps": 24.0,
+    "decode_backend": "torchcodec",
+    "return_uint8": true,
+    "center_crop": true,
+    "random_flip": false,
+    "num_workers": 6,
+    "prefetch_factor": 2,
+    "pin_memory": true,
+    "drop_last": true,
+    "max_samples": null,
+    "cache_dir": ".cache/omnivae",
+    "repeat": 1,
+    "recaption_prob": 0.0,
+    "captions_file": null,
+    "prompt_templates": [
+      "a high-quality photo of {label}"
+    ],
+    "use_full_label": false,
+    "metadata_paths": [],
+    "dataset_base_path": "/",
+    "sample_rate": 48000,
+    "num_audio_samples": 242000,
+    "max_num_audio_samples": 1440000,
+    "mono": true,
+    "append_duration_suffix": false,
+    "duration_precision": 1,
+    "path_field": "video_path",
+    "prompt_field": "av_caption",
+    "sources": [
+      {
+        "name": "nonspeech_1000w_g2",
+        "path": "data/train/nonspeech_1000w_group2_9376485.jsonl",
+        "weight": 0.5
+      },
+      {
+        "name": "speech_1000w_g2",
+        "path": "data/train/youtube_speech_10042814.jsonl",
+        "weight": 0.35
+      },
+      {
+        "name": "documentary",
+        "path": "data/train/documentary_2627844.jsonl",
+        "weight": 0.12
+      },
+      {
+        "name": "movies",
+        "path": "data/train/movies_8964574.jsonl",
+        "weight": 0.05
+      },
+      {
+        "name": "chinese_drama",
+        "path": "data/train/chinese_drama_3264550.jsonl",
+        "weight": 0.03
+      }
+    ],
+    "task_prefix_enabled": true,
+    "min_clip_duration_ratio": 0.99
+  },
+  "text_encoder": {
+    "model_name_or_path": "models/text_encoder/Qwen3.5-0.8B-Base",
+    "trust_remote_code": false,
+    "torch_dtype": "bfloat16",
+    "attn_implementation": "sdpa",
+    "disable_qwen3_5_fast_path": true,
+    "cache_enabled": false,
+    "compile_model": true,
+    "max_sequence_length": 512,
+    "local_files_only": true
+  },
+  "scheduler": {
+    "model_name_or_path": "models/dit/t2av/t2av_recon_avclip",
+    "subfolder": "scheduler",
+    "local_files_only": true,
+    "use_dynamic_shifting": false
+  },
+  "transformer": {
+    "arch": "zimage",
+    "backbone_name_or_path": "Qwen/Qwen3-VL-2B-Instruct",
+    "backbone_torch_dtype": "bfloat16",
+    "init_from_pretrained_backbone": true,
+    "text_config": null,
+    "trust_remote_code": false,
+    "local_files_only": false,
+    "attn_implementation": "flex_attention",
+    "max_sequence_length": 512,
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 16,
+    "dim": 2048,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 16,
+    "n_kv_heads": 16,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      1024,
+      512,
+      512
+    ],
+    "compile_model": true,
+    "attention_backend": null,
+    "bridge_interval": 1,
+    "bridge_enabled": true,
+    "use_asymmetric_ati": false,
+    "a2v_window_size": 1,
+    "compile_bridges": true,
+    "compile_mode": "default",
+    "compile_fullgraph": false,
+    "compile_dynamic": null
+  },
+  "validation": {
+    "modes": [
+      "audio_only",
+      "video_only",
+      "joint_av"
+    ],
+    "joint_av_prompts": {
+      "name": "versebench",
+      "jsonl_path": "eval/data/t2av/versebench_minimal/versebench_t2av_infer_minimal.jsonl",
+      "text_field": "av_caption",
+      "type_field": "type",
+      "index_field": "index",
+      "task_kind": "t2av",
+      "append_duration_suffix": false,
+      "max_examples": null
+    },
+    "video_only_prompts": {
+      "name": "t2v_inline",
+      "max_examples": null,
+      "prompts": [
+        "A cinematic documentary shot of a three-dimensional topographical map, clouds drifting over islands, somber historical lighting.",
+        "A tense cinematic scene inside a dimly lit historical palace hall, guards rushing through a wooden corridor, dramatic torchlight.",
+        "A close-up cinematic drama shot of a young woman lit by cool blue light, anxious expression, soft early 2000s television style.",
+        "A cinematic wide shot of ocean waves rolling onto a sandy beach at sunset, realistic lighting, gentle camera movement.",
+        "A young woman laughing while chatting with a friend at a sunny outdoor cafe.",
+        "An old man telling a story to a group of children sitting around him in a park.",
+        "A street musician playing guitar and singing on a busy city sidewalk.",
+        "A golden retriever running across a grassy field on a bright afternoon.",
+        "Cherry blossoms gently falling along a quiet path in spring.",
+        "A bowl of steaming noodles on a wooden table in a cozy small restaurant."
+      ]
+    },
+    "audio_only_prompt_sets": [
+      {
+        "name": "basetts_valid",
+        "task_kind": "tts",
+        "jsonl_path": "examples/eval/t2a_tts_basetts_sample.jsonl",
+        "text_field": "text",
+        "type_field": "type",
+        "index_field": "index",
+        "append_duration_suffix": true,
+        "duration_precision": 1,
+        "max_examples": 8
+      },
+      {
+        "name": "tta_general_en",
+        "task_kind": "tta",
+        "jsonl_path": "examples/eval/t2a_tta_general_en.jsonl",
+        "text_field": "text",
+        "type_field": "type",
+        "index_field": "index",
+        "append_duration_suffix": true,
+        "duration_precision": 1,
+        "max_examples": null
+      }
+    ],
+    "num_inference_steps": 50,
+    "video_guidance_scale": 4.0,
+    "audio_guidance_scale": 4.0,
+    "cfg_normalization": false,
+    "max_wandb_samples_per_source": 10,
+    "cfg_modes": [
+      "simple",
+      "dual"
+    ],
+    "video_text_guidance": 4.0,
+    "video_modality_guidance": 4.0,
+    "audio_text_guidance": 4.0,
+    "audio_modality_guidance": 4.0,
+    "video_frame_size": [
+      256,
+      256
+    ],
+    "video_num_frames": 121,
+    "video_fps": 24.0,
+    "audio_duration_seconds": 5.0417
+  },
+  "vae": {
+    "type": "univae",
+    "model_name_or_path": "models/vae/audio_video/recon_avclip",
+    "torch_dtype": "bfloat16",
+    "local_files_only": true,
+    "wan_chunk_mode": "parallel",
+    "compile_encode": true,
+    "compile_encode_mode": "reduce-overhead",
+    "compile_encode_fullgraph": false,
+    "branch": "video",
+    "use_ema": false
+  },
+  "audio_vae": {
+    "type": "univae",
+    "model_path": "models/vae/audio_only/recon_avclip_ft_decoder",
+    "hop_length": 960,
+    "latent_channels": 128,
+    "torch_dtype": "float32",
+    "compile_encode": true,
+    "compile_encode_fullgraph": false,
+    "compile_encode_mode": "reduce-overhead",
+    "branch": "audio",
+    "use_ema": false
+  },
+  "transformer_video": {
+    "arch": "zimage",
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 48,
+    "dim": 1920,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 15,
+    "n_kv_heads": 15,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      1024,
+      512,
+      512
+    ],
+    "compile_model": true,
+    "attention_backend": null
+  },
+  "transformer_audio": {
+    "arch": "zimage",
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 128,
+    "dim": 1920,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 15,
+    "n_kv_heads": 15,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      2048,
+      16,
+      16
+    ],
+    "compile_model": true,
+    "attention_backend": null
+  },
+  "train": {
+    "seed": 42,
+    "validation_steps": 5000,
+    "validation_prompts": [
+      "a high-quality photo of a goldfish",
+      "a high-quality photo of a tiger shark",
+      "a high-quality photo of an ostrich",
+      "a high-quality photo of a tabby cat"
+    ],
+    "validation_num_inference_steps": 20,
+    "validation_guidance_scale": 4.0,
+    "validation_cfg_normalization": false,
+    "validation_cfg_truncation": 1.0,
+    "validation_num_frames": null,
+    "validation_frame_size": null,
+    "validation_fps": null,
+    "validation_negative_prompts": null,
+    "shift_v": 5.0,
+    "shift_a": 3.0,
+    "validation_force_steps": [
+      0
+    ]
+  }
+}

models/dit/t2av/t2av_recon_avclip/resolved_config.yaml

@@ -0,0 +1,290 @@
+experiment:
+  name: t2av_recon_avclip
+  output_dir: models/dit/t2av/t2av_recon_avclip
+dataset:
+  type: av_paired_jsonl
+  root: omitted_public_release_path
+  meta_path: null
+  jsonl_index_path: null
+  jsonl_path_field: video_path
+  jsonl_prompt_field: prompt_v2
+  jsonl_prompt_index_path: null
+  slave_path: ''
+  base_image_path: ''
+  split: train
+  image_size: 256
+  frame_size:
+  - 256
+  - 256
+  num_frames: 121
+  target_fps: 24.0
+  decode_backend: torchcodec
+  return_uint8: true
+  center_crop: true
+  random_flip: false
+  num_workers: 6
+  prefetch_factor: 2
+  pin_memory: true
+  drop_last: true
+  max_samples: null
+  cache_dir: .cache/omnivae
+  repeat: 1
+  recaption_prob: 0.0
+  captions_file: null
+  prompt_templates:
+  - a high-quality photo of {label}
+  use_full_label: false
+  metadata_paths: []
+  dataset_base_path: /
+  sample_rate: 48000
+  num_audio_samples: 242000
+  max_num_audio_samples: 1440000
+  mono: true
+  append_duration_suffix: false
+  duration_precision: 1
+  path_field: video_path
+  prompt_field: av_caption
+  sources:
+  - name: nonspeech_1000w_g2
+    path: data/train/nonspeech_1000w_group2_9376485.jsonl
+    weight: 0.5
+  - name: speech_1000w_g2
+    path: data/train/youtube_speech_10042814.jsonl
+    weight: 0.35
+  - name: documentary
+    path: data/train/documentary_2627844.jsonl
+    weight: 0.12
+  - name: movies
+    path: data/train/movies_8964574.jsonl
+    weight: 0.05
+  - name: chinese_drama
+    path: data/train/chinese_drama_3264550.jsonl
+    weight: 0.03
+  task_prefix_enabled: true
+  min_clip_duration_ratio: 0.99
+text_encoder:
+  model_name_or_path: models/text_encoder/Qwen3.5-0.8B-Base
+  trust_remote_code: false
+  torch_dtype: bfloat16
+  attn_implementation: sdpa
+  disable_qwen3_5_fast_path: true
+  cache_enabled: false
+  compile_model: true
+  max_sequence_length: 512
+  local_files_only: true
+scheduler:
+  model_name_or_path: models/dit/t2av/t2av_recon_avclip
+  subfolder: scheduler
+  local_files_only: true
+  use_dynamic_shifting: false
+transformer:
+  arch: zimage
+  backbone_name_or_path: Qwen/Qwen3-VL-2B-Instruct
+  backbone_torch_dtype: bfloat16
+  init_from_pretrained_backbone: true
+  text_config: null
+  trust_remote_code: false
+  local_files_only: false
+  attn_implementation: flex_attention
+  max_sequence_length: 512
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 16
+  dim: 2048
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 16
+  n_kv_heads: 16
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 1024
+  - 512
+  - 512
+  compile_model: true
+  attention_backend: null
+  bridge_interval: 1
+  bridge_enabled: true
+  use_asymmetric_ati: false
+  a2v_window_size: 1
+  compile_bridges: true
+  compile_mode: default
+  compile_fullgraph: false
+  compile_dynamic: null
+validation:
+  modes:
+  - audio_only
+  - video_only
+  - joint_av
+  joint_av_prompts:
+    name: versebench
+    jsonl_path: eval/data/t2av/versebench_minimal/versebench_t2av_infer_minimal.jsonl
+    text_field: av_caption
+    type_field: type
+    index_field: index
+    task_kind: t2av
+    append_duration_suffix: false
+    max_examples: null
+  video_only_prompts:
+    name: t2v_inline
+    max_examples: null
+    prompts:
+    - A cinematic documentary shot of a three-dimensional topographical map, clouds
+      drifting over islands, somber historical lighting.
+    - A tense cinematic scene inside a dimly lit historical palace hall, guards rushing
+      through a wooden corridor, dramatic torchlight.
+    - A close-up cinematic drama shot of a young woman lit by cool blue light, anxious
+      expression, soft early 2000s television style.
+    - A cinematic wide shot of ocean waves rolling onto a sandy beach at sunset, realistic
+      lighting, gentle camera movement.
+    - A young woman laughing while chatting with a friend at a sunny outdoor cafe.
+    - An old man telling a story to a group of children sitting around him in a park.
+    - A street musician playing guitar and singing on a busy city sidewalk.
+    - A golden retriever running across a grassy field on a bright afternoon.
+    - Cherry blossoms gently falling along a quiet path in spring.
+    - A bowl of steaming noodles on a wooden table in a cozy small restaurant.
+  audio_only_prompt_sets:
+  - name: basetts_valid
+    task_kind: tts
+    jsonl_path: examples/eval/t2a_tts_basetts_sample.jsonl
+    text_field: text
+    type_field: type
+    index_field: index
+    append_duration_suffix: true
+    duration_precision: 1
+    max_examples: 8
+  - name: tta_general_en
+    task_kind: tta
+    jsonl_path: examples/eval/t2a_tta_general_en.jsonl
+    text_field: text
+    type_field: type
+    index_field: index
+    append_duration_suffix: true
+    duration_precision: 1
+    max_examples: null
+  num_inference_steps: 50
+  video_guidance_scale: 4.0
+  audio_guidance_scale: 4.0
+  cfg_normalization: false
+  max_wandb_samples_per_source: 10
+  cfg_modes:
+  - simple
+  - dual
+  video_text_guidance: 4.0
+  video_modality_guidance: 4.0
+  audio_text_guidance: 4.0
+  audio_modality_guidance: 4.0
+  video_frame_size:
+  - 256
+  - 256
+  video_num_frames: 121
+  video_fps: 24.0
+  audio_duration_seconds: 5.0417
+vae:
+  type: univae
+  model_name_or_path: models/vae/audio_video/recon_avclip
+  torch_dtype: bfloat16
+  local_files_only: true
+  wan_chunk_mode: parallel
+  compile_encode: true
+  compile_encode_mode: reduce-overhead
+  compile_encode_fullgraph: false
+  branch: video
+  use_ema: false
+audio_vae:
+  type: univae
+  model_path: models/vae/audio_only/recon_avclip_ft_decoder
+  hop_length: 960
+  latent_channels: 128
+  torch_dtype: float32
+  compile_encode: true
+  compile_encode_fullgraph: false
+  compile_encode_mode: reduce-overhead
+  branch: audio
+  use_ema: false
+transformer_video:
+  arch: zimage
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 48
+  dim: 1920
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 15
+  n_kv_heads: 15
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 1024
+  - 512
+  - 512
+  compile_model: true
+  attention_backend: null
+transformer_audio:
+  arch: zimage
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 128
+  dim: 1920
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 15
+  n_kv_heads: 15
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 2048
+  - 16
+  - 16
+  compile_model: true
+  attention_backend: null
+train:
+  seed: 42
+  validation_steps: 5000
+  validation_prompts:
+  - a high-quality photo of a goldfish
+  - a high-quality photo of a tiger shark
+  - a high-quality photo of an ostrich
+  - a high-quality photo of a tabby cat
+  validation_num_inference_steps: 20
+  validation_guidance_scale: 4.0
+  validation_cfg_normalization: false
+  validation_cfg_truncation: 1.0
+  validation_num_frames: null
+  validation_frame_size: null
+  validation_fps: null
+  validation_negative_prompts: null
+  shift_v: 5.0
+  shift_a: 3.0
+  validation_force_steps:
+  - 0

models/dit/t2av/t2av_recon_distill/bridges/bridge_config.json

@@ -0,0 +1,9 @@
+{
+  "bridge_interval": 1,
+  "use_asymmetric_ati": false,
+  "a2v_window_size": 1,
+  "n_bridges": 18,
+  "dim": 1920,
+  "n_heads": 15,
+  "head_dim": 128
+}

models/dit/t2av/t2av_recon_distill/metadata.json

@@ -0,0 +1,29 @@
+{
+  "export_format": "omnivae_generation_inference_package_v1",
+  "inference_only": true,
+  "task": "t2av",
+  "name": "t2av_recon_distill",
+  "source_experiment": "2_t2av_recon_distill_lr2",
+  "exported_at_utc": "2026-06-07T21:30:31+00:00",
+  "text_encoder_model_name_or_path": "models/text_encoder/Qwen3.5-0.8B-Base",
+  "vae_model_name_or_path": "models/vae/video_only/recon_distill",
+  "required_subdirs": [
+    "transformer_video",
+    "transformer_audio",
+    "bridges",
+    "tokenizer",
+    "scheduler"
+  ],
+  "video_vae_model_name_or_path": "models/vae/video_only/recon_distill",
+  "audio_vae_model_path": "models/vae/audio_only/recon_distill",
+  "joint_av": true,
+  "release_vae_paths_rewritten": true,
+  "release_experiment_family": "recon_distill",
+  "release_path_note": "VAE paths are relative to OMNIVAE_RELEASE_ROOT/models/vae. AVCLIP audio branches use ft_decoder checkpoints.",
+  "source_step": 200000,
+  "release_checkpoint_step": 200000,
+  "release_laion_checkpoint_name": "checkpoint-00200000",
+  "source_experiment_original_name": "2_t2av_recon_distill_lr2",
+  "source_checkpoint_provenance": "internal training checkpoint path omitted; use release_checkpoint_step and this package directory.",
+  "release_package_path": "models/dit/t2av/t2av_recon_distill"
+}

models/dit/t2av/t2av_recon_distill/resolved_config.json

@@ -0,0 +1,337 @@
+{
+  "experiment": {
+    "name": "t2av_recon_distill",
+    "output_dir": "models/dit/t2av/t2av_recon_distill"
+  },
+  "dataset": {
+    "type": "av_paired_jsonl",
+    "root": "omitted_public_release_path",
+    "meta_path": null,
+    "jsonl_index_path": null,
+    "jsonl_path_field": "video_path",
+    "jsonl_prompt_field": "prompt_v2",
+    "jsonl_prompt_index_path": null,
+    "slave_path": "",
+    "base_image_path": "",
+    "split": "train",
+    "image_size": 256,
+    "frame_size": [
+      256,
+      256
+    ],
+    "num_frames": 121,
+    "target_fps": 24.0,
+    "decode_backend": "torchcodec",
+    "return_uint8": true,
+    "center_crop": true,
+    "random_flip": false,
+    "num_workers": 6,
+    "prefetch_factor": 2,
+    "pin_memory": true,
+    "drop_last": true,
+    "max_samples": null,
+    "cache_dir": ".cache/omnivae",
+    "repeat": 1,
+    "recaption_prob": 0.0,
+    "captions_file": null,
+    "prompt_templates": [
+      "a high-quality photo of {label}"
+    ],
+    "use_full_label": false,
+    "metadata_paths": [],
+    "dataset_base_path": "/",
+    "sample_rate": 48000,
+    "num_audio_samples": 242000,
+    "max_num_audio_samples": 1440000,
+    "mono": true,
+    "append_duration_suffix": false,
+    "duration_precision": 1,
+    "path_field": "video_path",
+    "prompt_field": "av_caption",
+    "sources": [
+      {
+        "name": "nonspeech_1000w_g2",
+        "path": "data/train/nonspeech_1000w_group2_9376485.jsonl",
+        "weight": 0.5
+      },
+      {
+        "name": "speech_1000w_g2",
+        "path": "data/train/youtube_speech_10042814.jsonl",
+        "weight": 0.35
+      },
+      {
+        "name": "documentary",
+        "path": "data/train/documentary_2627844.jsonl",
+        "weight": 0.12
+      },
+      {
+        "name": "movies",
+        "path": "data/train/movies_8964574.jsonl",
+        "weight": 0.05
+      },
+      {
+        "name": "chinese_drama",
+        "path": "data/train/chinese_drama_3264550.jsonl",
+        "weight": 0.03
+      }
+    ],
+    "task_prefix_enabled": true,
+    "min_clip_duration_ratio": 0.99
+  },
+  "text_encoder": {
+    "model_name_or_path": "models/text_encoder/Qwen3.5-0.8B-Base",
+    "trust_remote_code": false,
+    "torch_dtype": "bfloat16",
+    "attn_implementation": "sdpa",
+    "disable_qwen3_5_fast_path": true,
+    "cache_enabled": false,
+    "compile_model": true,
+    "max_sequence_length": 512,
+    "local_files_only": true
+  },
+  "scheduler": {
+    "model_name_or_path": "models/dit/t2av/t2av_recon_distill",
+    "subfolder": "scheduler",
+    "local_files_only": true,
+    "use_dynamic_shifting": false
+  },
+  "transformer": {
+    "arch": "zimage",
+    "backbone_name_or_path": "Qwen/Qwen3-VL-2B-Instruct",
+    "backbone_torch_dtype": "bfloat16",
+    "init_from_pretrained_backbone": true,
+    "text_config": null,
+    "trust_remote_code": false,
+    "local_files_only": false,
+    "attn_implementation": "flex_attention",
+    "max_sequence_length": 512,
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 16,
+    "dim": 2048,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 16,
+    "n_kv_heads": 16,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      1024,
+      512,
+      512
+    ],
+    "compile_model": true,
+    "attention_backend": null,
+    "bridge_interval": 1,
+    "bridge_enabled": true,
+    "use_asymmetric_ati": false,
+    "a2v_window_size": 1,
+    "compile_bridges": true,
+    "compile_mode": "default",
+    "compile_fullgraph": false,
+    "compile_dynamic": null
+  },
+  "validation": {
+    "modes": [
+      "audio_only",
+      "video_only",
+      "joint_av"
+    ],
+    "joint_av_prompts": {
+      "name": "versebench",
+      "jsonl_path": "eval/data/t2av/versebench_minimal/versebench_t2av_infer_minimal.jsonl",
+      "text_field": "av_caption",
+      "type_field": "type",
+      "index_field": "index",
+      "task_kind": "t2av",
+      "append_duration_suffix": false,
+      "max_examples": null
+    },
+    "video_only_prompts": {
+      "name": "t2v_inline",
+      "max_examples": null,
+      "prompts": [
+        "A cinematic documentary shot of a three-dimensional topographical map, clouds drifting over islands, somber historical lighting.",
+        "A tense cinematic scene inside a dimly lit historical palace hall, guards rushing through a wooden corridor, dramatic torchlight.",
+        "A close-up cinematic drama shot of a young woman lit by cool blue light, anxious expression, soft early 2000s television style.",
+        "A cinematic wide shot of ocean waves rolling onto a sandy beach at sunset, realistic lighting, gentle camera movement.",
+        "A young woman laughing while chatting with a friend at a sunny outdoor cafe.",
+        "An old man telling a story to a group of children sitting around him in a park.",
+        "A street musician playing guitar and singing on a busy city sidewalk.",
+        "A golden retriever running across a grassy field on a bright afternoon.",
+        "Cherry blossoms gently falling along a quiet path in spring.",
+        "A bowl of steaming noodles on a wooden table in a cozy small restaurant."
+      ]
+    },
+    "audio_only_prompt_sets": [
+      {
+        "name": "basetts_valid",
+        "task_kind": "tts",
+        "jsonl_path": "examples/eval/t2a_tts_basetts_sample.jsonl",
+        "text_field": "text",
+        "type_field": "type",
+        "index_field": "index",
+        "append_duration_suffix": true,
+        "duration_precision": 1,
+        "max_examples": 8
+      },
+      {
+        "name": "tta_general_en",
+        "task_kind": "tta",
+        "jsonl_path": "examples/eval/t2a_tta_general_en.jsonl",
+        "text_field": "text",
+        "type_field": "type",
+        "index_field": "index",
+        "append_duration_suffix": true,
+        "duration_precision": 1,
+        "max_examples": null
+      }
+    ],
+    "num_inference_steps": 50,
+    "video_guidance_scale": 4.0,
+    "audio_guidance_scale": 4.0,
+    "cfg_normalization": false,
+    "max_wandb_samples_per_source": 10,
+    "cfg_modes": [
+      "simple",
+      "dual"
+    ],
+    "video_text_guidance": 4.0,
+    "video_modality_guidance": 4.0,
+    "audio_text_guidance": 4.0,
+    "audio_modality_guidance": 4.0,
+    "video_frame_size": [
+      256,
+      256
+    ],
+    "video_num_frames": 121,
+    "video_fps": 24.0,
+    "audio_duration_seconds": 5.0417
+  },
+  "vae": {
+    "type": "univae",
+    "model_name_or_path": "models/vae/video_only/recon_distill",
+    "torch_dtype": "bfloat16",
+    "local_files_only": true,
+    "wan_chunk_mode": "parallel",
+    "compile_encode": true,
+    "compile_encode_mode": "reduce-overhead",
+    "compile_encode_fullgraph": false,
+    "branch": "video",
+    "use_ema": false
+  },
+  "audio_vae": {
+    "type": "univae",
+    "model_path": "models/vae/audio_only/recon_distill",
+    "hop_length": 960,
+    "latent_channels": 128,
+    "torch_dtype": "float32",
+    "compile_encode": true,
+    "compile_encode_fullgraph": false,
+    "compile_encode_mode": "reduce-overhead",
+    "branch": "audio",
+    "use_ema": false
+  },
+  "transformer_video": {
+    "arch": "zimage",
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 48,
+    "dim": 1920,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 15,
+    "n_kv_heads": 15,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      1024,
+      512,
+      512
+    ],
+    "compile_model": true,
+    "attention_backend": null
+  },
+  "transformer_audio": {
+    "arch": "zimage",
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 128,
+    "dim": 1920,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 15,
+    "n_kv_heads": 15,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      2048,
+      16,
+      16
+    ],
+    "compile_model": true,
+    "attention_backend": null
+  },
+  "train": {
+    "seed": 42,
+    "validation_steps": 5000,
+    "validation_prompts": [
+      "a high-quality photo of a goldfish",
+      "a high-quality photo of a tiger shark",
+      "a high-quality photo of an ostrich",
+      "a high-quality photo of a tabby cat"
+    ],
+    "validation_num_inference_steps": 20,
+    "validation_guidance_scale": 4.0,
+    "validation_cfg_normalization": false,
+    "validation_cfg_truncation": 1.0,
+    "validation_num_frames": null,
+    "validation_frame_size": null,
+    "validation_fps": null,
+    "validation_negative_prompts": null,
+    "shift_v": 5.0,
+    "shift_a": 3.0,
+    "validation_force_steps": [
+      0
+    ]
+  }
+}

models/dit/t2av/t2av_recon_distill/resolved_config.yaml

@@ -0,0 +1,290 @@
+experiment:
+  name: t2av_recon_distill
+  output_dir: models/dit/t2av/t2av_recon_distill
+dataset:
+  type: av_paired_jsonl
+  root: omitted_public_release_path
+  meta_path: null
+  jsonl_index_path: null
+  jsonl_path_field: video_path
+  jsonl_prompt_field: prompt_v2
+  jsonl_prompt_index_path: null
+  slave_path: ''
+  base_image_path: ''
+  split: train
+  image_size: 256
+  frame_size:
+  - 256
+  - 256
+  num_frames: 121
+  target_fps: 24.0
+  decode_backend: torchcodec
+  return_uint8: true
+  center_crop: true
+  random_flip: false
+  num_workers: 6
+  prefetch_factor: 2
+  pin_memory: true
+  drop_last: true
+  max_samples: null
+  cache_dir: .cache/omnivae
+  repeat: 1
+  recaption_prob: 0.0
+  captions_file: null
+  prompt_templates:
+  - a high-quality photo of {label}
+  use_full_label: false
+  metadata_paths: []
+  dataset_base_path: /
+  sample_rate: 48000
+  num_audio_samples: 242000
+  max_num_audio_samples: 1440000
+  mono: true
+  append_duration_suffix: false
+  duration_precision: 1
+  path_field: video_path
+  prompt_field: av_caption
+  sources:
+  - name: nonspeech_1000w_g2
+    path: data/train/nonspeech_1000w_group2_9376485.jsonl
+    weight: 0.5
+  - name: speech_1000w_g2
+    path: data/train/youtube_speech_10042814.jsonl
+    weight: 0.35
+  - name: documentary
+    path: data/train/documentary_2627844.jsonl
+    weight: 0.12
+  - name: movies
+    path: data/train/movies_8964574.jsonl
+    weight: 0.05
+  - name: chinese_drama
+    path: data/train/chinese_drama_3264550.jsonl
+    weight: 0.03
+  task_prefix_enabled: true
+  min_clip_duration_ratio: 0.99
+text_encoder:
+  model_name_or_path: models/text_encoder/Qwen3.5-0.8B-Base
+  trust_remote_code: false
+  torch_dtype: bfloat16
+  attn_implementation: sdpa
+  disable_qwen3_5_fast_path: true
+  cache_enabled: false
+  compile_model: true
+  max_sequence_length: 512
+  local_files_only: true
+scheduler:
+  model_name_or_path: models/dit/t2av/t2av_recon_distill
+  subfolder: scheduler
+  local_files_only: true
+  use_dynamic_shifting: false
+transformer:
+  arch: zimage
+  backbone_name_or_path: Qwen/Qwen3-VL-2B-Instruct
+  backbone_torch_dtype: bfloat16
+  init_from_pretrained_backbone: true
+  text_config: null
+  trust_remote_code: false
+  local_files_only: false
+  attn_implementation: flex_attention
+  max_sequence_length: 512
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 16
+  dim: 2048
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 16
+  n_kv_heads: 16
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 1024
+  - 512
+  - 512
+  compile_model: true
+  attention_backend: null
+  bridge_interval: 1
+  bridge_enabled: true
+  use_asymmetric_ati: false
+  a2v_window_size: 1
+  compile_bridges: true
+  compile_mode: default
+  compile_fullgraph: false
+  compile_dynamic: null
+validation:
+  modes:
+  - audio_only
+  - video_only
+  - joint_av
+  joint_av_prompts:
+    name: versebench
+    jsonl_path: eval/data/t2av/versebench_minimal/versebench_t2av_infer_minimal.jsonl
+    text_field: av_caption
+    type_field: type
+    index_field: index
+    task_kind: t2av
+    append_duration_suffix: false
+    max_examples: null
+  video_only_prompts:
+    name: t2v_inline
+    max_examples: null
+    prompts:
+    - A cinematic documentary shot of a three-dimensional topographical map, clouds
+      drifting over islands, somber historical lighting.
+    - A tense cinematic scene inside a dimly lit historical palace hall, guards rushing
+      through a wooden corridor, dramatic torchlight.
+    - A close-up cinematic drama shot of a young woman lit by cool blue light, anxious
+      expression, soft early 2000s television style.
+    - A cinematic wide shot of ocean waves rolling onto a sandy beach at sunset, realistic
+      lighting, gentle camera movement.
+    - A young woman laughing while chatting with a friend at a sunny outdoor cafe.
+    - An old man telling a story to a group of children sitting around him in a park.
+    - A street musician playing guitar and singing on a busy city sidewalk.
+    - A golden retriever running across a grassy field on a bright afternoon.
+    - Cherry blossoms gently falling along a quiet path in spring.
+    - A bowl of steaming noodles on a wooden table in a cozy small restaurant.
+  audio_only_prompt_sets:
+  - name: basetts_valid
+    task_kind: tts
+    jsonl_path: examples/eval/t2a_tts_basetts_sample.jsonl
+    text_field: text
+    type_field: type
+    index_field: index
+    append_duration_suffix: true
+    duration_precision: 1
+    max_examples: 8
+  - name: tta_general_en
+    task_kind: tta
+    jsonl_path: examples/eval/t2a_tta_general_en.jsonl
+    text_field: text
+    type_field: type
+    index_field: index
+    append_duration_suffix: true
+    duration_precision: 1
+    max_examples: null
+  num_inference_steps: 50
+  video_guidance_scale: 4.0
+  audio_guidance_scale: 4.0
+  cfg_normalization: false
+  max_wandb_samples_per_source: 10
+  cfg_modes:
+  - simple
+  - dual
+  video_text_guidance: 4.0
+  video_modality_guidance: 4.0
+  audio_text_guidance: 4.0
+  audio_modality_guidance: 4.0
+  video_frame_size:
+  - 256
+  - 256
+  video_num_frames: 121
+  video_fps: 24.0
+  audio_duration_seconds: 5.0417
+vae:
+  type: univae
+  model_name_or_path: models/vae/video_only/recon_distill
+  torch_dtype: bfloat16
+  local_files_only: true
+  wan_chunk_mode: parallel
+  compile_encode: true
+  compile_encode_mode: reduce-overhead
+  compile_encode_fullgraph: false
+  branch: video
+  use_ema: false
+audio_vae:
+  type: univae
+  model_path: models/vae/audio_only/recon_distill
+  hop_length: 960
+  latent_channels: 128
+  torch_dtype: float32
+  compile_encode: true
+  compile_encode_fullgraph: false
+  compile_encode_mode: reduce-overhead
+  branch: audio
+  use_ema: false
+transformer_video:
+  arch: zimage
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 48
+  dim: 1920
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 15
+  n_kv_heads: 15
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 1024
+  - 512
+  - 512
+  compile_model: true
+  attention_backend: null
+transformer_audio:
+  arch: zimage
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 128
+  dim: 1920
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 15
+  n_kv_heads: 15
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 2048
+  - 16
+  - 16
+  compile_model: true
+  attention_backend: null
+train:
+  seed: 42
+  validation_steps: 5000
+  validation_prompts:
+  - a high-quality photo of a goldfish
+  - a high-quality photo of a tiger shark
+  - a high-quality photo of an ostrich
+  - a high-quality photo of a tabby cat
+  validation_num_inference_steps: 20
+  validation_guidance_scale: 4.0
+  validation_cfg_normalization: false
+  validation_cfg_truncation: 1.0
+  validation_num_frames: null
+  validation_frame_size: null
+  validation_fps: null
+  validation_negative_prompts: null
+  shift_v: 5.0
+  shift_a: 3.0
+  validation_force_steps:
+  - 0

models/dit/t2av/t2av_recon_distill/scheduler/scheduler_config.json

@@ -0,0 +1,18 @@
+{
+  "_class_name": "FlowMatchEulerDiscreteScheduler",
+  "_diffusers_version": "0.37.1",
+  "base_image_seq_len": 256,
+  "base_shift": 0.5,
+  "invert_sigmas": false,
+  "max_image_seq_len": 4096,
+  "max_shift": 1.15,
+  "num_train_timesteps": 1000,
+  "shift": 3.0,
+  "shift_terminal": null,
+  "stochastic_sampling": false,
+  "time_shift_type": "exponential",
+  "use_beta_sigmas": false,
+  "use_dynamic_shifting": false,
+  "use_exponential_sigmas": false,
+  "use_karras_sigmas": false
+}

models/dit/t2av/t2av_recon_distill/tokenizer/chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is true %}
+        {{- '<think>\n' }}
+    {%- else %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

models/dit/t2av/t2av_recon_distill/tokenizer/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": false,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "split_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

models/dit/t2av/t2av_recon_distill/transformer_audio/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "ZImageTransformer2DModel",
+  "_diffusers_version": "0.37.1",
+  "all_f_patch_size": [
+    1
+  ],
+  "all_patch_size": [
+    1
+  ],
+  "axes_dims": [
+    16,
+    56,
+    56
+  ],
+  "axes_lens": [
+    2048,
+    16,
+    16
+  ],
+  "cap_feat_dim": 1024,
+  "dim": 1920,
+  "in_channels": 128,
+  "n_heads": 15,
+  "n_kv_heads": 15,
+  "n_layers": 18,
+  "n_refiner_layers": 2,
+  "norm_eps": 1e-05,
+  "qk_norm": true,
+  "rope_theta": 256.0,
+  "siglip_feat_dim": null,
+  "t_scale": 1000.0
+}

models/dit/t2av/t2av_recon_distill/transformer_video/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "ZImageTransformer2DModel",
+  "_diffusers_version": "0.37.1",
+  "all_f_patch_size": [
+    1
+  ],
+  "all_patch_size": [
+    1
+  ],
+  "axes_dims": [
+    16,
+    56,
+    56
+  ],
+  "axes_lens": [
+    1024,
+    512,
+    512
+  ],
+  "cap_feat_dim": 1024,
+  "dim": 1920,
+  "in_channels": 48,
+  "n_heads": 15,
+  "n_kv_heads": 15,
+  "n_layers": 18,
+  "n_refiner_layers": 2,
+  "norm_eps": 1e-05,
+  "qk_norm": true,
+  "rope_theta": 256.0,
+  "siglip_feat_dim": null,
+  "t_scale": 1000.0
+}

models/dit/t2av/t2av_recon_distill_avclip/bridges/bridge_config.json

@@ -0,0 +1,9 @@
+{
+  "bridge_interval": 1,
+  "use_asymmetric_ati": false,
+  "a2v_window_size": 1,
+  "n_bridges": 18,
+  "dim": 1920,
+  "n_heads": 15,
+  "head_dim": 128
+}

models/dit/t2av/t2av_recon_distill_avclip/metadata.json

@@ -0,0 +1,29 @@
+{
+  "export_format": "omnivae_generation_inference_package_v1",
+  "inference_only": true,
+  "task": "t2av",
+  "name": "t2av_recon_distill_avclip",
+  "source_experiment": "2_t2av_recon_distill_avclip_lr2",
+  "exported_at_utc": "2026-06-07T21:30:31+00:00",
+  "text_encoder_model_name_or_path": "models/text_encoder/Qwen3.5-0.8B-Base",
+  "vae_model_name_or_path": "models/vae/audio_video/recon_distill_avclip",
+  "required_subdirs": [
+    "transformer_video",
+    "transformer_audio",
+    "bridges",
+    "tokenizer",
+    "scheduler"
+  ],
+  "video_vae_model_name_or_path": "models/vae/audio_video/recon_distill_avclip",
+  "audio_vae_model_path": "models/vae/audio_only/recon_distill_avclip_ft_decoder",
+  "joint_av": true,
+  "release_vae_paths_rewritten": true,
+  "release_experiment_family": "recon_distill_avclip",
+  "release_path_note": "VAE paths are relative to OMNIVAE_RELEASE_ROOT/models/vae. AVCLIP audio branches use ft_decoder checkpoints.",
+  "source_step": 200000,
+  "release_checkpoint_step": 200000,
+  "release_laion_checkpoint_name": "checkpoint-00200000",
+  "source_experiment_original_name": "2_t2av_recon_distill_avclip_lr2",
+  "source_checkpoint_provenance": "internal training checkpoint path omitted; use release_checkpoint_step and this package directory.",
+  "release_package_path": "models/dit/t2av/t2av_recon_distill_avclip"
+}

models/dit/t2av/t2av_recon_distill_avclip/resolved_config.json

@@ -0,0 +1,337 @@
+{
+  "experiment": {
+    "name": "t2av_recon_distill_avclip",
+    "output_dir": "models/dit/t2av/t2av_recon_distill_avclip"
+  },
+  "dataset": {
+    "type": "av_paired_jsonl",
+    "root": "omitted_public_release_path",
+    "meta_path": null,
+    "jsonl_index_path": null,
+    "jsonl_path_field": "video_path",
+    "jsonl_prompt_field": "prompt_v2",
+    "jsonl_prompt_index_path": null,
+    "slave_path": "",
+    "base_image_path": "",
+    "split": "train",
+    "image_size": 256,
+    "frame_size": [
+      256,
+      256
+    ],
+    "num_frames": 121,
+    "target_fps": 24.0,
+    "decode_backend": "torchcodec",
+    "return_uint8": true,
+    "center_crop": true,
+    "random_flip": false,
+    "num_workers": 6,
+    "prefetch_factor": 2,
+    "pin_memory": true,
+    "drop_last": true,
+    "max_samples": null,
+    "cache_dir": ".cache/omnivae",
+    "repeat": 1,
+    "recaption_prob": 0.0,
+    "captions_file": null,
+    "prompt_templates": [
+      "a high-quality photo of {label}"
+    ],
+    "use_full_label": false,
+    "metadata_paths": [],
+    "dataset_base_path": "/",
+    "sample_rate": 48000,
+    "num_audio_samples": 242000,
+    "max_num_audio_samples": 1440000,
+    "mono": true,
+    "append_duration_suffix": false,
+    "duration_precision": 1,
+    "path_field": "video_path",
+    "prompt_field": "av_caption",
+    "sources": [
+      {
+        "name": "nonspeech_1000w_g2",
+        "path": "data/train/nonspeech_1000w_group2_9376485.jsonl",
+        "weight": 0.5
+      },
+      {
+        "name": "speech_1000w_g2",
+        "path": "data/train/youtube_speech_10042814.jsonl",
+        "weight": 0.35
+      },
+      {
+        "name": "documentary",
+        "path": "data/train/documentary_2627844.jsonl",
+        "weight": 0.12
+      },
+      {
+        "name": "movies",
+        "path": "data/train/movies_8964574.jsonl",
+        "weight": 0.05
+      },
+      {
+        "name": "chinese_drama",
+        "path": "data/train/chinese_drama_3264550.jsonl",
+        "weight": 0.03
+      }
+    ],
+    "task_prefix_enabled": true,
+    "min_clip_duration_ratio": 0.99
+  },
+  "text_encoder": {
+    "model_name_or_path": "models/text_encoder/Qwen3.5-0.8B-Base",
+    "trust_remote_code": false,
+    "torch_dtype": "bfloat16",
+    "attn_implementation": "sdpa",
+    "disable_qwen3_5_fast_path": true,
+    "cache_enabled": false,
+    "compile_model": true,
+    "max_sequence_length": 512,
+    "local_files_only": true
+  },
+  "scheduler": {
+    "model_name_or_path": "models/dit/t2av/t2av_recon_distill_avclip",
+    "subfolder": "scheduler",
+    "local_files_only": true,
+    "use_dynamic_shifting": false
+  },
+  "transformer": {
+    "arch": "zimage",
+    "backbone_name_or_path": "Qwen/Qwen3-VL-2B-Instruct",
+    "backbone_torch_dtype": "bfloat16",
+    "init_from_pretrained_backbone": true,
+    "text_config": null,
+    "trust_remote_code": false,
+    "local_files_only": false,
+    "attn_implementation": "flex_attention",
+    "max_sequence_length": 512,
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 16,
+    "dim": 2048,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 16,
+    "n_kv_heads": 16,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      1024,
+      512,
+      512
+    ],
+    "compile_model": true,
+    "attention_backend": null,
+    "bridge_interval": 1,
+    "bridge_enabled": true,
+    "use_asymmetric_ati": false,
+    "a2v_window_size": 1,
+    "compile_bridges": true,
+    "compile_mode": "default",
+    "compile_fullgraph": false,
+    "compile_dynamic": null
+  },
+  "validation": {
+    "modes": [
+      "audio_only",
+      "video_only",
+      "joint_av"
+    ],
+    "joint_av_prompts": {
+      "name": "versebench",
+      "jsonl_path": "eval/data/t2av/versebench_minimal/versebench_t2av_infer_minimal.jsonl",
+      "text_field": "av_caption",
+      "type_field": "type",
+      "index_field": "index",
+      "task_kind": "t2av",
+      "append_duration_suffix": false,
+      "max_examples": null
+    },
+    "video_only_prompts": {
+      "name": "t2v_inline",
+      "max_examples": null,
+      "prompts": [
+        "A cinematic documentary shot of a three-dimensional topographical map, clouds drifting over islands, somber historical lighting.",
+        "A tense cinematic scene inside a dimly lit historical palace hall, guards rushing through a wooden corridor, dramatic torchlight.",
+        "A close-up cinematic drama shot of a young woman lit by cool blue light, anxious expression, soft early 2000s television style.",
+        "A cinematic wide shot of ocean waves rolling onto a sandy beach at sunset, realistic lighting, gentle camera movement.",
+        "A young woman laughing while chatting with a friend at a sunny outdoor cafe.",
+        "An old man telling a story to a group of children sitting around him in a park.",
+        "A street musician playing guitar and singing on a busy city sidewalk.",
+        "A golden retriever running across a grassy field on a bright afternoon.",
+        "Cherry blossoms gently falling along a quiet path in spring.",
+        "A bowl of steaming noodles on a wooden table in a cozy small restaurant."
+      ]
+    },
+    "audio_only_prompt_sets": [
+      {
+        "name": "basetts_valid",
+        "task_kind": "tts",
+        "jsonl_path": "examples/eval/t2a_tts_basetts_sample.jsonl",
+        "text_field": "text",
+        "type_field": "type",
+        "index_field": "index",
+        "append_duration_suffix": true,
+        "duration_precision": 1,
+        "max_examples": 8
+      },
+      {
+        "name": "tta_general_en",
+        "task_kind": "tta",
+        "jsonl_path": "examples/eval/t2a_tta_general_en.jsonl",
+        "text_field": "text",
+        "type_field": "type",
+        "index_field": "index",
+        "append_duration_suffix": true,
+        "duration_precision": 1,
+        "max_examples": null
+      }
+    ],
+    "num_inference_steps": 50,
+    "video_guidance_scale": 4.0,
+    "audio_guidance_scale": 4.0,
+    "cfg_normalization": false,
+    "max_wandb_samples_per_source": 10,
+    "cfg_modes": [
+      "simple",
+      "dual"
+    ],
+    "video_text_guidance": 4.0,
+    "video_modality_guidance": 4.0,
+    "audio_text_guidance": 4.0,
+    "audio_modality_guidance": 4.0,
+    "video_frame_size": [
+      256,
+      256
+    ],
+    "video_num_frames": 121,
+    "video_fps": 24.0,
+    "audio_duration_seconds": 5.0417
+  },
+  "vae": {
+    "type": "univae",
+    "model_name_or_path": "models/vae/audio_video/recon_distill_avclip",
+    "torch_dtype": "bfloat16",
+    "local_files_only": true,
+    "wan_chunk_mode": "parallel",
+    "compile_encode": true,
+    "compile_encode_mode": "reduce-overhead",
+    "compile_encode_fullgraph": false,
+    "branch": "video",
+    "use_ema": false
+  },
+  "audio_vae": {
+    "type": "univae",
+    "model_path": "models/vae/audio_only/recon_distill_avclip_ft_decoder",
+    "hop_length": 960,
+    "latent_channels": 128,
+    "torch_dtype": "float32",
+    "compile_encode": true,
+    "compile_encode_fullgraph": false,
+    "compile_encode_mode": "reduce-overhead",
+    "branch": "audio",
+    "use_ema": false
+  },
+  "transformer_video": {
+    "arch": "zimage",
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 48,
+    "dim": 1920,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 15,
+    "n_kv_heads": 15,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      1024,
+      512,
+      512
+    ],
+    "compile_model": true,
+    "attention_backend": null
+  },
+  "transformer_audio": {
+    "arch": "zimage",
+    "all_patch_size": [
+      1
+    ],
+    "all_f_patch_size": [
+      1
+    ],
+    "in_channels": 128,
+    "dim": 1920,
+    "n_layers": 18,
+    "n_refiner_layers": 2,
+    "n_heads": 15,
+    "n_kv_heads": 15,
+    "norm_eps": 1e-05,
+    "qk_norm": true,
+    "rope_theta": 256.0,
+    "t_scale": 1000.0,
+    "use_timestep": true,
+    "predict_target": "v",
+    "axes_dims": [
+      16,
+      56,
+      56
+    ],
+    "axes_lens": [
+      2048,
+      16,
+      16
+    ],
+    "compile_model": true,
+    "attention_backend": null
+  },
+  "train": {
+    "seed": 42,
+    "validation_steps": 5000,
+    "validation_prompts": [
+      "a high-quality photo of a goldfish",
+      "a high-quality photo of a tiger shark",
+      "a high-quality photo of an ostrich",
+      "a high-quality photo of a tabby cat"
+    ],
+    "validation_num_inference_steps": 20,
+    "validation_guidance_scale": 4.0,
+    "validation_cfg_normalization": false,
+    "validation_cfg_truncation": 1.0,
+    "validation_num_frames": null,
+    "validation_frame_size": null,
+    "validation_fps": null,
+    "validation_negative_prompts": null,
+    "shift_v": 5.0,
+    "shift_a": 3.0,
+    "validation_force_steps": [
+      0
+    ]
+  }
+}

models/dit/t2av/t2av_recon_distill_avclip/resolved_config.yaml

@@ -0,0 +1,290 @@
+experiment:
+  name: t2av_recon_distill_avclip
+  output_dir: models/dit/t2av/t2av_recon_distill_avclip
+dataset:
+  type: av_paired_jsonl
+  root: omitted_public_release_path
+  meta_path: null
+  jsonl_index_path: null
+  jsonl_path_field: video_path
+  jsonl_prompt_field: prompt_v2
+  jsonl_prompt_index_path: null
+  slave_path: ''
+  base_image_path: ''
+  split: train
+  image_size: 256
+  frame_size:
+  - 256
+  - 256
+  num_frames: 121
+  target_fps: 24.0
+  decode_backend: torchcodec
+  return_uint8: true
+  center_crop: true
+  random_flip: false
+  num_workers: 6
+  prefetch_factor: 2
+  pin_memory: true
+  drop_last: true
+  max_samples: null
+  cache_dir: .cache/omnivae
+  repeat: 1
+  recaption_prob: 0.0
+  captions_file: null
+  prompt_templates:
+  - a high-quality photo of {label}
+  use_full_label: false
+  metadata_paths: []
+  dataset_base_path: /
+  sample_rate: 48000
+  num_audio_samples: 242000
+  max_num_audio_samples: 1440000
+  mono: true
+  append_duration_suffix: false
+  duration_precision: 1
+  path_field: video_path
+  prompt_field: av_caption
+  sources:
+  - name: nonspeech_1000w_g2
+    path: data/train/nonspeech_1000w_group2_9376485.jsonl
+    weight: 0.5
+  - name: speech_1000w_g2
+    path: data/train/youtube_speech_10042814.jsonl
+    weight: 0.35
+  - name: documentary
+    path: data/train/documentary_2627844.jsonl
+    weight: 0.12
+  - name: movies
+    path: data/train/movies_8964574.jsonl
+    weight: 0.05
+  - name: chinese_drama
+    path: data/train/chinese_drama_3264550.jsonl
+    weight: 0.03
+  task_prefix_enabled: true
+  min_clip_duration_ratio: 0.99
+text_encoder:
+  model_name_or_path: models/text_encoder/Qwen3.5-0.8B-Base
+  trust_remote_code: false
+  torch_dtype: bfloat16
+  attn_implementation: sdpa
+  disable_qwen3_5_fast_path: true
+  cache_enabled: false
+  compile_model: true
+  max_sequence_length: 512
+  local_files_only: true
+scheduler:
+  model_name_or_path: models/dit/t2av/t2av_recon_distill_avclip
+  subfolder: scheduler
+  local_files_only: true
+  use_dynamic_shifting: false
+transformer:
+  arch: zimage
+  backbone_name_or_path: Qwen/Qwen3-VL-2B-Instruct
+  backbone_torch_dtype: bfloat16
+  init_from_pretrained_backbone: true
+  text_config: null
+  trust_remote_code: false
+  local_files_only: false
+  attn_implementation: flex_attention
+  max_sequence_length: 512
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 16
+  dim: 2048
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 16
+  n_kv_heads: 16
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 1024
+  - 512
+  - 512
+  compile_model: true
+  attention_backend: null
+  bridge_interval: 1
+  bridge_enabled: true
+  use_asymmetric_ati: false
+  a2v_window_size: 1
+  compile_bridges: true
+  compile_mode: default
+  compile_fullgraph: false
+  compile_dynamic: null
+validation:
+  modes:
+  - audio_only
+  - video_only
+  - joint_av
+  joint_av_prompts:
+    name: versebench
+    jsonl_path: eval/data/t2av/versebench_minimal/versebench_t2av_infer_minimal.jsonl
+    text_field: av_caption
+    type_field: type
+    index_field: index
+    task_kind: t2av
+    append_duration_suffix: false
+    max_examples: null
+  video_only_prompts:
+    name: t2v_inline
+    max_examples: null
+    prompts:
+    - A cinematic documentary shot of a three-dimensional topographical map, clouds
+      drifting over islands, somber historical lighting.
+    - A tense cinematic scene inside a dimly lit historical palace hall, guards rushing
+      through a wooden corridor, dramatic torchlight.
+    - A close-up cinematic drama shot of a young woman lit by cool blue light, anxious
+      expression, soft early 2000s television style.
+    - A cinematic wide shot of ocean waves rolling onto a sandy beach at sunset, realistic
+      lighting, gentle camera movement.
+    - A young woman laughing while chatting with a friend at a sunny outdoor cafe.
+    - An old man telling a story to a group of children sitting around him in a park.
+    - A street musician playing guitar and singing on a busy city sidewalk.
+    - A golden retriever running across a grassy field on a bright afternoon.
+    - Cherry blossoms gently falling along a quiet path in spring.
+    - A bowl of steaming noodles on a wooden table in a cozy small restaurant.
+  audio_only_prompt_sets:
+  - name: basetts_valid
+    task_kind: tts
+    jsonl_path: examples/eval/t2a_tts_basetts_sample.jsonl
+    text_field: text
+    type_field: type
+    index_field: index
+    append_duration_suffix: true
+    duration_precision: 1
+    max_examples: 8
+  - name: tta_general_en
+    task_kind: tta
+    jsonl_path: examples/eval/t2a_tta_general_en.jsonl
+    text_field: text
+    type_field: type
+    index_field: index
+    append_duration_suffix: true
+    duration_precision: 1
+    max_examples: null
+  num_inference_steps: 50
+  video_guidance_scale: 4.0
+  audio_guidance_scale: 4.0
+  cfg_normalization: false
+  max_wandb_samples_per_source: 10
+  cfg_modes:
+  - simple
+  - dual
+  video_text_guidance: 4.0
+  video_modality_guidance: 4.0
+  audio_text_guidance: 4.0
+  audio_modality_guidance: 4.0
+  video_frame_size:
+  - 256
+  - 256
+  video_num_frames: 121
+  video_fps: 24.0
+  audio_duration_seconds: 5.0417
+vae:
+  type: univae
+  model_name_or_path: models/vae/audio_video/recon_distill_avclip
+  torch_dtype: bfloat16
+  local_files_only: true
+  wan_chunk_mode: parallel
+  compile_encode: true
+  compile_encode_mode: reduce-overhead
+  compile_encode_fullgraph: false
+  branch: video
+  use_ema: false
+audio_vae:
+  type: univae
+  model_path: models/vae/audio_only/recon_distill_avclip_ft_decoder
+  hop_length: 960
+  latent_channels: 128
+  torch_dtype: float32
+  compile_encode: true
+  compile_encode_fullgraph: false
+  compile_encode_mode: reduce-overhead
+  branch: audio
+  use_ema: false
+transformer_video:
+  arch: zimage
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 48
+  dim: 1920
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 15
+  n_kv_heads: 15
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 1024
+  - 512
+  - 512
+  compile_model: true
+  attention_backend: null
+transformer_audio:
+  arch: zimage
+  all_patch_size:
+  - 1
+  all_f_patch_size:
+  - 1
+  in_channels: 128
+  dim: 1920
+  n_layers: 18
+  n_refiner_layers: 2
+  n_heads: 15
+  n_kv_heads: 15
+  norm_eps: 1.0e-05
+  qk_norm: true
+  rope_theta: 256.0
+  t_scale: 1000.0
+  use_timestep: true
+  predict_target: v
+  axes_dims:
+  - 16
+  - 56
+  - 56
+  axes_lens:
+  - 2048
+  - 16
+  - 16
+  compile_model: true
+  attention_backend: null
+train:
+  seed: 42
+  validation_steps: 5000
+  validation_prompts:
+  - a high-quality photo of a goldfish
+  - a high-quality photo of a tiger shark
+  - a high-quality photo of an ostrich
+  - a high-quality photo of a tabby cat
+  validation_num_inference_steps: 20
+  validation_guidance_scale: 4.0
+  validation_cfg_normalization: false
+  validation_cfg_truncation: 1.0
+  validation_num_frames: null
+  validation_frame_size: null
+  validation_fps: null
+  validation_negative_prompts: null
+  shift_v: 5.0
+  shift_a: 3.0
+  validation_force_steps:
+  - 0

models/dit/t2av/t2av_recon_distill_avclip/scheduler/scheduler_config.json

@@ -0,0 +1,18 @@
+{
+  "_class_name": "FlowMatchEulerDiscreteScheduler",
+  "_diffusers_version": "0.37.1",
+  "base_image_seq_len": 256,
+  "base_shift": 0.5,
+  "invert_sigmas": false,
+  "max_image_seq_len": 4096,
+  "max_shift": 1.15,
+  "num_train_timesteps": 1000,
+  "shift": 3.0,
+  "shift_terminal": null,
+  "stochastic_sampling": false,
+  "time_shift_type": "exponential",
+  "use_beta_sigmas": false,
+  "use_dynamic_shifting": false,
+  "use_exponential_sigmas": false,
+  "use_karras_sigmas": false
+}

models/dit/t2av/t2av_recon_distill_avclip/tokenizer/chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is true %}
+        {{- '<think>\n' }}
+    {%- else %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- endif %}
+{%- endif %}

models/dit/t2av/t2av_recon_distill_avclip/tokenizer/tokenizer_config.json

@@ -0,0 +1,31 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "image_token": "<|image_pad|>",
+  "is_local": false,
+  "model_max_length": 262144,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "split_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}

models/dit/t2av/t2av_recon_distill_avclip/transformer_audio/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "ZImageTransformer2DModel",
+  "_diffusers_version": "0.37.1",
+  "all_f_patch_size": [
+    1
+  ],
+  "all_patch_size": [
+    1
+  ],
+  "axes_dims": [
+    16,
+    56,
+    56
+  ],
+  "axes_lens": [
+    2048,
+    16,
+    16
+  ],
+  "cap_feat_dim": 1024,
+  "dim": 1920,
+  "in_channels": 128,
+  "n_heads": 15,
+  "n_kv_heads": 15,
+  "n_layers": 18,
+  "n_refiner_layers": 2,
+  "norm_eps": 1e-05,
+  "qk_norm": true,
+  "rope_theta": 256.0,
+  "siglip_feat_dim": null,
+  "t_scale": 1000.0
+}

models/dit/t2av/t2av_recon_distill_avclip/transformer_video/config.json

@@ -0,0 +1,32 @@
+{
+  "_class_name": "ZImageTransformer2DModel",
+  "_diffusers_version": "0.37.1",
+  "all_f_patch_size": [
+    1
+  ],
+  "all_patch_size": [
+    1
+  ],
+  "axes_dims": [
+    16,
+    56,
+    56
+  ],
+  "axes_lens": [
+    1024,
+    512,
+    512
+  ],
+  "cap_feat_dim": 1024,
+  "dim": 1920,
+  "in_channels": 48,
+  "n_heads": 15,
+  "n_kv_heads": 15,
+  "n_layers": 18,
+  "n_refiner_layers": 2,
+  "norm_eps": 1e-05,
+  "qk_norm": true,
+  "rope_theta": 256.0,
+  "siglip_feat_dim": null,
+  "t_scale": 1000.0
+}

models/vae/audio_only/recon/config.yaml

@@ -0,0 +1,254 @@
+seed: 1234
+audio_sample_rate: 48000
+num_frames: 121
+resolution: 256
+target_fps: 24
+video_frame_sample_rate: 1
+model:
+  video:
+    model_name: WanVAE22
+    model_config: scripts/causalvae/wanvae22_48dim.json
+    pretrained_model_name_or_path: null
+  audio:
+    audio_sample_rate: 48000
+    sample_rate: 48000
+    encoder_dim: 128
+    encoder_rates:
+    - 2
+    - 3
+    - 4
+    - 5
+    - 8
+    latent_dim: 128
+    decoder_dim: 2048
+    decoder_rates:
+    - 8
+    - 5
+    - 4
+    - 3
+    - 2
+    n_codebooks: 9
+    codebook_size: 1024
+    codebook_dim: 8
+    quantizer_dropout: false
+    continuous: true
+    pretrained_model_name_or_path: null
+  contrastive:
+    contrastive_use_mean: true
+    embed_dim: 512
+    segment_hidden_dim: 1024
+    global_hidden_dim: 1024
+    segment_count: null
+    init_scale: 0.07
+    clamp_scale_min: 0.001
+    clamp_scale_max: 0.5
+    gather_for_loss: true
+    num_negatives: 23
+    num_negative_videos: 63
+    use_sdpa: true
+    spatial_pool_mode: mean
+    spatial_merge_factor: 4
+    transformer_dim: 768
+    spatial_transformer_layers: 1
+    segment_transformer_layers: 1
+    global_transformer_layers: 1
+    use_segment_loss: false
+    use_global_loss: false
+    enabled: false
+  distill:
+    enabled: false
+    video_target_dim: 1152
+    image_target_dim: 1152
+    audio_target_dim: 1280
+    video_spatial_factor: 4
+    image_spatial_factor: 4
+    temporal_agg_ratio: 3
+    audio_distill_type: d_axis
+    distill_proj_type: conv
+    distill_use_conv3d: false
+    distill_proj_layers: 2
+    distill_proj_hidden_dim: 256
+    distill_proj_before_agg: true
+    distill_dim_schedule: fixed
+    distill_use_sampled: false
+    audio_proj_type: conv
+    audio_proj_layers: 2
+    audio_proj_hidden_dim: 512
+  llm:
+    enabled: false
+    llm_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+    freeze_llm: false
+    include_first_frame: false
+    spatial_merge_factor: 4
+    temporal_merge_factor: 1
+    patch_embed_dim: 256
+    patch_stride: 2
+    spatial_merge_size: 2
+    audio_ds_hidden: 256
+    audio_ds_stride: 2
+    audio_inter_dim: 512
+data:
+  train:
+    metadata_paths:
+      train_main: omitted_public_release_path
+    metadata_weights:
+      train_main: 1.0
+    use_file_processor: true
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    num_workers: 4
+    prefetch_factor: 2
+  val_video:
+    video_dir: omitted_public_release_path
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    max_samples: 1000
+    batch_size: 1
+  val_audio:
+    audio_dir: omitted_public_release_path
+    audio_sample_rate: 48000
+    max_duration: 5.0
+    max_samples: 1000
+    batch_size: 1
+  val_contrastive:
+    jsonl_paths:
+      vggsound_valid: data/train/vggsound_valid_filtered.jsonl
+      ct_nonspeech: data/train/nonspeech_valid_10000.jsonl
+      ct_speech: data/train/speech_valid_10000.jsonl
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    max_samples: 20000
+    batch_size: 1
+    val_segment_num_negatives:
+    - 29
+    - 64
+    val_segment_num_negative_videos: ${model.contrastive.num_negative_videos}
+    val_global_num_negatives:
+    - 64
+    - 256
+training:
+  max_steps: 500000
+  batch_size: 4
+  lr: 1.0e-05
+  weight_decay: 0.0001
+  betas:
+  - 0.9
+  - 0.999
+  max_grad_norm: 1.0
+  gradient_accumulation_steps: 1
+  dtype: fp32
+  video_vae_dtype: fp32
+  audio_vae_dtype: fp32
+  contrastive_dtype: bfloat16
+  llm_dtype: bfloat16
+  gradient_checkpointing: false
+  random_start: false
+  use_ema: false
+  ema_decay: 0.999
+  qk_norm: false
+  log_steps: 3
+  stdout_steps: 10
+  eval_steps: 2000
+  save_steps: 2000
+  max_checkpoints: 5000
+loss:
+  use_video_recon: false
+  use_audio_recon: true
+  use_segment_contrastive: false
+  use_global_contrastive: false
+  lambda_group_video: 1.0
+  lambda_group_audio: 1.0
+  lambda_group_contrastive: 1.0
+  lambda_video_recon: 1.0
+  lambda_video_kl: 0.0165
+  lambda_video_lpips: 1.0
+  video_loss_reduction: sum
+  lambda_audio_mel: 1.0
+  lambda_audio_recon: 0.0
+  lambda_audio_kl: 0.5
+  lambda_segment_contrastive: 1.0
+  lambda_global_contrastive: 1.0
+  global_contrastive_start_steps: 0
+  adaptive_loss_balance: true
+  adaptive_balance_audio_ratio: 0.5
+  adaptive_balance_contrastive_ratio: 0.5
+  adaptive_loss_balance_by_uncertainty: false
+  uncertainty_warmup_steps: 100
+  use_semantic_distill: false
+  encoder_fps: 4
+  encoder_resolution: 256
+  distill_vision_layer: 18
+  distill_audio_layer: 24
+  lambda_distill_image_cosine: 1.0
+  lambda_distill_image_distance: 1.0
+  lambda_distill_video_cosine: 1.0
+  lambda_distill_video_distance: 1.0
+  lambda_distill_audio_d_axis: 120.0
+  lambda_distill_audio_t_axis: 1.0
+  distill_w_hyper: 0.1
+  distill_audio_type: d_axis
+  lambda_group_distill: 1.0
+  distill_spatial_norm: true
+  distill_spatial_norm_gamma: 0.6
+  distill_margin_cosine: 0.0
+  distill_margin_distance: 0.25
+  distill_use_dist_matrix: false
+  adaptive_distill_balance: false
+  adaptive_distill_video_ratio: 0.1
+  adaptive_distill_audio_ratio: 0.1
+  use_llm_caption: false
+  lambda_group_llm: 1.0
+  lambda_llm_caption: 1.0
+  mel_loss_kwargs:
+    sample_rate: 48000
+    n_mels:
+    - 5
+    - 10
+    - 20
+    - 40
+    - 80
+    - 160
+    - 320
+    window_lengths:
+    - 32
+    - 64
+    - 128
+    - 256
+    - 512
+    - 1024
+    - 2048
+    weights:
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    pow: 1.0
+    mel_fmin:
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    mel_fmax:
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+output:
+  exp_root: outputs

models/vae/audio_only/recon/metadata.json

@@ -0,0 +1,8 @@
+{
+  "package_type": "vae",
+  "modality": "audio_only",
+  "name": "recon",
+  "config": "models/vae/audio_only/recon/config.yaml",
+  "checkpoint": "models/vae/audio_only/recon/state_dict.pt",
+  "notes": "Audio reconstruction VAE checkpoint without training state."
+}

models/vae/audio_only/recon_avclip_ft_decoder/config.yaml

@@ -0,0 +1,259 @@
+seed: 1234
+audio_sample_rate: 48000
+num_frames: 121
+resolution: 256
+target_fps: 24
+video_frame_sample_rate: 1
+model:
+  video:
+    model_name: WanVAE22
+    model_config: scripts/causalvae/wanvae22_48dim.json
+    pretrained_model_name_or_path: null
+  audio:
+    audio_sample_rate: 48000
+    sample_rate: 48000
+    encoder_dim: 128
+    encoder_rates:
+    - 2
+    - 3
+    - 4
+    - 5
+    - 8
+    latent_dim: 128
+    decoder_dim: 2048
+    decoder_rates:
+    - 8
+    - 5
+    - 4
+    - 3
+    - 2
+    n_codebooks: 9
+    codebook_size: 1024
+    codebook_dim: 8
+    quantizer_dropout: false
+    continuous: true
+    pretrained_model_name_or_path: null
+  contrastive:
+    contrastive_use_mean: true
+    embed_dim: 512
+    segment_hidden_dim: 1024
+    global_hidden_dim: 1024
+    segment_count: null
+    init_scale: 0.07
+    clamp_scale_min: 0.001
+    clamp_scale_max: 0.5
+    gather_for_loss: true
+    num_negatives: 23
+    num_negative_videos: 63
+    use_sdpa: true
+    spatial_pool_mode: mean
+    spatial_merge_factor: 4
+    transformer_dim: 768
+    spatial_transformer_layers: 1
+    segment_transformer_layers: 1
+    global_transformer_layers: 1
+    use_segment_loss: false
+    use_global_loss: false
+    enabled: false
+  distill:
+    enabled: false
+    video_target_dim: 1152
+    image_target_dim: 1152
+    audio_target_dim: 1280
+    video_spatial_factor: 4
+    image_spatial_factor: 4
+    temporal_agg_ratio: 3
+    audio_distill_type: d_axis
+    distill_proj_type: conv
+    distill_use_conv3d: false
+    distill_proj_layers: 2
+    distill_proj_hidden_dim: 256
+    distill_proj_before_agg: true
+    distill_dim_schedule: fixed
+    distill_use_sampled: false
+    audio_proj_type: conv
+    audio_proj_layers: 2
+    audio_proj_hidden_dim: 512
+  llm:
+    enabled: false
+    llm_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+    freeze_llm: false
+    include_first_frame: false
+    spatial_merge_factor: 4
+    temporal_merge_factor: 1
+    patch_embed_dim: 256
+    patch_stride: 2
+    spatial_merge_size: 2
+    audio_ds_hidden: 256
+    audio_ds_stride: 2
+    audio_inter_dim: 512
+data:
+  train:
+    metadata_paths:
+      train_main: omitted_public_release_path
+    metadata_weights:
+      train_main: 1.0
+    use_file_processor: true
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    num_workers: 4
+    prefetch_factor: 2
+  val_video:
+    video_dir: omitted_public_release_path
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    max_samples: 1000
+    batch_size: 1
+  val_audio:
+    audio_dir: omitted_public_release_path
+    audio_sample_rate: 48000
+    max_duration: 5.0
+    max_samples: 1000
+    batch_size: 1
+  val_contrastive:
+    jsonl_paths:
+      vggsound_valid: data/train/vggsound_valid_filtered.jsonl
+      ct_nonspeech: data/train/nonspeech_valid_10000.jsonl
+      ct_speech: data/train/speech_valid_10000.jsonl
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    max_samples: 20000
+    batch_size: 1
+    val_segment_num_negatives:
+    - 29
+    - 64
+    val_segment_num_negative_videos: ${model.contrastive.num_negative_videos}
+    val_global_num_negatives:
+    - 64
+    - 256
+training:
+  max_steps: 500000
+  batch_size: 4
+  lr: 1.0e-05
+  weight_decay: 0.0001
+  betas:
+  - 0.9
+  - 0.999
+  max_grad_norm: 1.0
+  gradient_accumulation_steps: 1
+  dtype: fp32
+  video_vae_dtype: fp32
+  audio_vae_dtype: fp32
+  contrastive_dtype: bfloat16
+  llm_dtype: bfloat16
+  gradient_checkpointing: false
+  random_start: false
+  use_ema: false
+  ema_decay: 0.999
+  qk_norm: false
+  log_steps: 3
+  stdout_steps: 10
+  eval_steps: 2000
+  save_steps: 2000
+  max_checkpoints: 5000
+  lr_disc: 0.0005
+loss:
+  use_video_recon: false
+  use_audio_recon: true
+  use_segment_contrastive: false
+  use_global_contrastive: false
+  lambda_group_video: 1.0
+  lambda_group_audio: 1.0
+  lambda_group_contrastive: 1.0
+  lambda_video_recon: 1.0
+  lambda_video_kl: 0.0165
+  lambda_video_lpips: 1.0
+  video_loss_reduction: sum
+  lambda_audio_mel: 1.0
+  lambda_audio_recon: 0.0
+  lambda_audio_kl: 0.0
+  lambda_segment_contrastive: 1.0
+  lambda_global_contrastive: 1.0
+  global_contrastive_start_steps: 0
+  adaptive_loss_balance: true
+  adaptive_balance_audio_ratio: 0.5
+  adaptive_balance_contrastive_ratio: 0.5
+  adaptive_loss_balance_by_uncertainty: false
+  uncertainty_warmup_steps: 100
+  use_semantic_distill: false
+  encoder_fps: 4
+  encoder_resolution: 256
+  distill_vision_layer: 18
+  distill_audio_layer: 24
+  lambda_distill_image_cosine: 1.0
+  lambda_distill_image_distance: 1.0
+  lambda_distill_video_cosine: 1.0
+  lambda_distill_video_distance: 1.0
+  lambda_distill_audio_d_axis: 120.0
+  lambda_distill_audio_t_axis: 1.0
+  distill_w_hyper: 0.1
+  distill_audio_type: d_axis
+  lambda_group_distill: 1.0
+  distill_spatial_norm: true
+  distill_spatial_norm_gamma: 0.6
+  distill_margin_cosine: 0.0
+  distill_margin_distance: 0.25
+  distill_use_dist_matrix: false
+  adaptive_distill_balance: false
+  adaptive_distill_video_ratio: 0.1
+  adaptive_distill_audio_ratio: 0.1
+  use_llm_caption: false
+  lambda_group_llm: 1.0
+  lambda_llm_caption: 1.0
+  mel_loss_kwargs:
+    sample_rate: 48000
+    n_mels:
+    - 5
+    - 10
+    - 20
+    - 40
+    - 80
+    - 160
+    - 320
+    window_lengths:
+    - 32
+    - 64
+    - 128
+    - 256
+    - 512
+    - 1024
+    - 2048
+    weights:
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    pow: 1.0
+    mel_fmin:
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    mel_fmax:
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+  freeze_audio_encoder: true
+  use_audio_disc: true
+  lambda_audio_adv: 1.0
+  lambda_audio_feature_matching: 2.0
+output:
+  exp_root: outputs

models/vae/audio_only/recon_avclip_ft_decoder/metadata.json

@@ -0,0 +1,8 @@
+{
+  "package_type": "vae",
+  "modality": "audio_only",
+  "name": "recon_avclip_ft_decoder",
+  "config": "models/vae/audio_only/recon_avclip_ft_decoder/config.yaml",
+  "checkpoint": "models/vae/audio_only/recon_avclip_ft_decoder/state_dict.pt",
+  "notes": "Audio decoder fine-tuned VAE checkpoint used by T2A/T2AV recon_avclip."
+}

models/vae/audio_only/recon_distill/config.yaml

@@ -0,0 +1,259 @@
+seed: 1234
+audio_sample_rate: 48000
+num_frames: 121
+resolution: 256
+target_fps: 24
+video_frame_sample_rate: 1
+model:
+  video:
+    model_name: WanVAE22
+    model_config: scripts/causalvae/wanvae22_48dim.json
+    pretrained_model_name_or_path: null
+  audio:
+    audio_sample_rate: 48000
+    sample_rate: 48000
+    encoder_dim: 128
+    encoder_rates:
+    - 2
+    - 3
+    - 4
+    - 5
+    - 8
+    latent_dim: 128
+    decoder_dim: 2048
+    decoder_rates:
+    - 8
+    - 5
+    - 4
+    - 3
+    - 2
+    n_codebooks: 9
+    codebook_size: 1024
+    codebook_dim: 8
+    quantizer_dropout: false
+    continuous: true
+    pretrained_model_name_or_path: null
+  contrastive:
+    contrastive_use_mean: true
+    embed_dim: 512
+    segment_hidden_dim: 1024
+    global_hidden_dim: 1024
+    segment_count: null
+    init_scale: 0.07
+    clamp_scale_min: 0.001
+    clamp_scale_max: 0.5
+    gather_for_loss: true
+    num_negatives: 23
+    num_negative_videos: 63
+    use_sdpa: true
+    spatial_pool_mode: mean
+    spatial_merge_factor: 4
+    transformer_dim: 768
+    spatial_transformer_layers: 1
+    segment_transformer_layers: 1
+    global_transformer_layers: 1
+    use_segment_loss: false
+    use_global_loss: false
+    enabled: false
+  distill:
+    enabled: true
+    video_target_dim: 1152
+    image_target_dim: 1152
+    audio_target_dim: 1280
+    video_spatial_factor: 4
+    image_spatial_factor: 4
+    temporal_agg_ratio: 3
+    audio_distill_type: t_axis
+    distill_proj_type: conv
+    distill_use_conv3d: false
+    distill_proj_layers: 2
+    distill_proj_hidden_dim: 256
+    distill_proj_before_agg: true
+    distill_dim_schedule: doubling
+    distill_use_sampled: false
+    audio_proj_type: conv
+    audio_proj_layers: 2
+    audio_proj_hidden_dim: 512
+  llm:
+    enabled: false
+    llm_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+    freeze_llm: false
+    include_first_frame: false
+    spatial_merge_factor: 4
+    temporal_merge_factor: 1
+    patch_embed_dim: 256
+    patch_stride: 2
+    spatial_merge_size: 2
+    audio_ds_hidden: 256
+    audio_ds_stride: 2
+    audio_inter_dim: 512
+data:
+  train:
+    metadata_paths:
+      train_main: omitted_public_release_path
+    metadata_weights:
+      train_main: 1.0
+    use_file_processor: true
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    num_workers: 4
+    prefetch_factor: 2
+  val_video:
+    video_dir: omitted_public_release_path
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    max_samples: 1000
+    batch_size: 1
+  val_audio:
+    audio_dir: omitted_public_release_path
+    audio_sample_rate: 48000
+    max_duration: 5.0
+    max_samples: 1000
+    batch_size: 1
+  val_contrastive:
+    jsonl_paths:
+      vggsound_valid: data/train/vggsound_valid_filtered.jsonl
+      ct_nonspeech: data/train/nonspeech_valid_10000.jsonl
+      ct_speech: data/train/speech_valid_10000.jsonl
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    max_samples: 20000
+    batch_size: 1
+    val_segment_num_negatives:
+    - 29
+    - 64
+    val_segment_num_negative_videos: ${model.contrastive.num_negative_videos}
+    val_global_num_negatives:
+    - 64
+    - 256
+training:
+  max_steps: 500000
+  batch_size: 4
+  lr: 1.0e-05
+  weight_decay: 0.0001
+  betas:
+  - 0.9
+  - 0.999
+  max_grad_norm: 1.0
+  gradient_accumulation_steps: 1
+  dtype: fp32
+  video_vae_dtype: fp32
+  audio_vae_dtype: fp32
+  contrastive_dtype: bfloat16
+  llm_dtype: bfloat16
+  gradient_checkpointing: false
+  random_start: false
+  use_ema: false
+  ema_decay: 0.999
+  qk_norm: false
+  log_steps: 3
+  stdout_steps: 10
+  eval_steps: 2000
+  save_steps: 2000
+  max_checkpoints: 5000
+  lr_disc: 0.0001
+loss:
+  use_video_recon: false
+  use_audio_recon: true
+  use_segment_contrastive: false
+  use_global_contrastive: false
+  lambda_group_video: 1.0
+  lambda_group_audio: 1.0
+  lambda_group_contrastive: 1.0
+  lambda_video_recon: 1.0
+  lambda_video_kl: 0.0165
+  lambda_video_lpips: 1.0
+  video_loss_reduction: sum
+  lambda_audio_mel: 1.0
+  lambda_audio_recon: 0.0
+  lambda_audio_kl: 0.5
+  lambda_segment_contrastive: 1.0
+  lambda_global_contrastive: 1.0
+  global_contrastive_start_steps: 0
+  adaptive_loss_balance: true
+  adaptive_balance_audio_ratio: 0.5
+  adaptive_balance_contrastive_ratio: 0.5
+  adaptive_loss_balance_by_uncertainty: false
+  uncertainty_warmup_steps: 100
+  use_semantic_distill: true
+  encoder_fps: 4
+  encoder_resolution: 256
+  distill_vision_layer: 18
+  distill_audio_layer: 24
+  lambda_distill_image_cosine: 1.0
+  lambda_distill_image_distance: 1.0
+  lambda_distill_video_cosine: 1.0
+  lambda_distill_video_distance: 1.0
+  lambda_distill_audio_d_axis: 120.0
+  lambda_distill_audio_t_axis: 1.0
+  distill_w_hyper: 0.1
+  distill_audio_type: t_axis
+  lambda_group_distill: 1.0
+  distill_spatial_norm: true
+  distill_spatial_norm_gamma: 0.6
+  distill_margin_cosine: 0.0
+  distill_margin_distance: 0.25
+  distill_use_dist_matrix: false
+  adaptive_distill_balance: true
+  adaptive_distill_video_ratio: 0.1
+  adaptive_distill_audio_ratio: 1.0
+  use_llm_caption: false
+  lambda_group_llm: 1.0
+  lambda_llm_caption: 1.0
+  mel_loss_kwargs:
+    sample_rate: 48000
+    n_mels:
+    - 5
+    - 10
+    - 20
+    - 40
+    - 80
+    - 160
+    - 320
+    window_lengths:
+    - 32
+    - 64
+    - 128
+    - 256
+    - 512
+    - 1024
+    - 2048
+    weights:
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    pow: 1.0
+    mel_fmin:
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    mel_fmax:
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+  use_audio_disc: true
+  lambda_audio_adv: 1.0
+  lambda_audio_feature_matching: 1.0
+  semantic_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+output:
+  exp_root: outputs

models/vae/audio_only/recon_distill/metadata.json

@@ -0,0 +1,8 @@
+{
+  "package_type": "vae",
+  "modality": "audio_only",
+  "name": "recon_distill",
+  "config": "models/vae/audio_only/recon_distill/config.yaml",
+  "checkpoint": "models/vae/audio_only/recon_distill/state_dict.pt",
+  "notes": "Distilled audio reconstruction VAE checkpoint without training state."
+}

models/vae/audio_only/recon_distill_avclip_ft_decoder/config.yaml

@@ -0,0 +1,261 @@
+seed: 1234
+audio_sample_rate: 48000
+num_frames: 121
+resolution: 256
+target_fps: 24
+video_frame_sample_rate: 1
+model:
+  video:
+    model_name: WanVAE22
+    model_config: scripts/causalvae/wanvae22_48dim.json
+    pretrained_model_name_or_path: null
+  audio:
+    audio_sample_rate: 48000
+    sample_rate: 48000
+    encoder_dim: 128
+    encoder_rates:
+    - 2
+    - 3
+    - 4
+    - 5
+    - 8
+    latent_dim: 128
+    decoder_dim: 2048
+    decoder_rates:
+    - 8
+    - 5
+    - 4
+    - 3
+    - 2
+    n_codebooks: 9
+    codebook_size: 1024
+    codebook_dim: 8
+    quantizer_dropout: false
+    continuous: true
+    pretrained_model_name_or_path: null
+  contrastive:
+    contrastive_use_mean: true
+    embed_dim: 512
+    segment_hidden_dim: 1024
+    global_hidden_dim: 1024
+    segment_count: null
+    init_scale: 0.07
+    clamp_scale_min: 0.001
+    clamp_scale_max: 0.5
+    gather_for_loss: true
+    num_negatives: 23
+    num_negative_videos: 63
+    use_sdpa: true
+    spatial_pool_mode: mean
+    spatial_merge_factor: 4
+    transformer_dim: 768
+    spatial_transformer_layers: 1
+    segment_transformer_layers: 1
+    global_transformer_layers: 1
+    use_segment_loss: false
+    use_global_loss: false
+    enabled: false
+  distill:
+    enabled: false
+    video_target_dim: 1152
+    image_target_dim: 1152
+    audio_target_dim: 1280
+    video_spatial_factor: 4
+    image_spatial_factor: 4
+    temporal_agg_ratio: 3
+    audio_distill_type: d_axis
+    distill_proj_type: conv
+    distill_use_conv3d: false
+    distill_proj_layers: 2
+    distill_proj_hidden_dim: 256
+    distill_proj_before_agg: true
+    distill_dim_schedule: fixed
+    distill_use_sampled: false
+    audio_proj_type: conv
+    audio_proj_layers: 2
+    audio_proj_hidden_dim: 512
+  llm:
+    enabled: false
+    llm_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+    freeze_llm: false
+    include_first_frame: false
+    spatial_merge_factor: 4
+    temporal_merge_factor: 1
+    patch_embed_dim: 256
+    patch_stride: 2
+    spatial_merge_size: 2
+    audio_ds_hidden: 256
+    audio_ds_stride: 2
+    audio_inter_dim: 512
+data:
+  train:
+    metadata_paths:
+      train_main: omitted_public_release_path
+    metadata_weights:
+      train_main: 1.0
+    use_file_processor: true
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    num_workers: 4
+    prefetch_factor: 2
+  val_video:
+    jsonl_paths:
+      panda70m_test: data/train/panda70m-test.jsonl
+      ucf101_test: data/train/ucf101-test.jsonl
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    max_samples: 1000
+    batch_size: 1
+  val_audio:
+    audio_dir: omitted_public_release_path
+    audio_sample_rate: 48000
+    max_duration: 5.0
+    max_samples: 1000
+    batch_size: 1
+  val_contrastive:
+    jsonl_paths:
+      vggsound_valid: data/train/vggsound_valid_filtered.jsonl
+      ct_nonspeech: data/train/nonspeech_valid_10000.jsonl
+      ct_speech: data/train/speech_valid_10000.jsonl
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    max_samples: 20000
+    batch_size: 1
+    val_segment_num_negatives:
+    - 29
+    - 64
+    val_segment_num_negative_videos: ${model.contrastive.num_negative_videos}
+    val_global_num_negatives:
+    - 64
+    - 256
+training:
+  max_steps: 500000
+  batch_size: 4
+  lr: 1.0e-05
+  weight_decay: 0.0001
+  betas:
+  - 0.9
+  - 0.999
+  max_grad_norm: 1.0
+  gradient_accumulation_steps: 1
+  dtype: fp32
+  video_vae_dtype: fp32
+  audio_vae_dtype: fp32
+  contrastive_dtype: bfloat16
+  llm_dtype: bfloat16
+  gradient_checkpointing: false
+  random_start: false
+  use_ema: false
+  ema_decay: 0.999
+  qk_norm: false
+  log_steps: 3
+  stdout_steps: 10
+  eval_steps: 2000
+  save_steps: 2000
+  max_checkpoints: 5000
+  lr_disc: 0.0005
+loss:
+  use_video_recon: false
+  use_audio_recon: true
+  use_segment_contrastive: false
+  use_global_contrastive: false
+  lambda_group_video: 1.0
+  lambda_group_audio: 1.0
+  lambda_group_contrastive: 1.0
+  lambda_video_recon: 1.0
+  lambda_video_kl: 0.0165
+  lambda_video_lpips: 1.0
+  video_loss_reduction: sum
+  lambda_audio_mel: 1.0
+  lambda_audio_recon: 0.0
+  lambda_audio_kl: 0.0
+  lambda_segment_contrastive: 1.0
+  lambda_global_contrastive: 1.0
+  global_contrastive_start_steps: 0
+  adaptive_loss_balance: true
+  adaptive_balance_audio_ratio: 0.5
+  adaptive_balance_contrastive_ratio: 0.5
+  adaptive_loss_balance_by_uncertainty: false
+  uncertainty_warmup_steps: 100
+  use_semantic_distill: false
+  encoder_fps: 4
+  encoder_resolution: 256
+  distill_vision_layer: 18
+  distill_audio_layer: 24
+  lambda_distill_image_cosine: 1.0
+  lambda_distill_image_distance: 1.0
+  lambda_distill_video_cosine: 1.0
+  lambda_distill_video_distance: 1.0
+  lambda_distill_audio_d_axis: 120.0
+  lambda_distill_audio_t_axis: 1.0
+  distill_w_hyper: 0.1
+  distill_audio_type: d_axis
+  lambda_group_distill: 1.0
+  distill_spatial_norm: true
+  distill_spatial_norm_gamma: 0.6
+  distill_margin_cosine: 0.0
+  distill_margin_distance: 0.25
+  distill_use_dist_matrix: false
+  adaptive_distill_balance: false
+  adaptive_distill_video_ratio: 0.1
+  adaptive_distill_audio_ratio: 0.1
+  use_llm_caption: false
+  lambda_group_llm: 1.0
+  lambda_llm_caption: 1.0
+  mel_loss_kwargs:
+    sample_rate: 48000
+    n_mels:
+    - 5
+    - 10
+    - 20
+    - 40
+    - 80
+    - 160
+    - 320
+    window_lengths:
+    - 32
+    - 64
+    - 128
+    - 256
+    - 512
+    - 1024
+    - 2048
+    weights:
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    pow: 1.0
+    mel_fmin:
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    mel_fmax:
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+  freeze_audio_encoder: true
+  use_audio_disc: true
+  lambda_audio_adv: 1.0
+  lambda_audio_feature_matching: 2.0
+output:
+  exp_root: outputs

models/vae/audio_only/recon_distill_avclip_ft_decoder/metadata.json

@@ -0,0 +1,8 @@
+{
+  "package_type": "vae",
+  "modality": "audio_only",
+  "name": "recon_distill_avclip_ft_decoder",
+  "config": "models/vae/audio_only/recon_distill_avclip_ft_decoder/config.yaml",
+  "checkpoint": "models/vae/audio_only/recon_distill_avclip_ft_decoder/state_dict.pt",
+  "notes": "Audio decoder fine-tuned VAE checkpoint used by T2A/T2AV recon_distill_avclip."
+}

models/vae/audio_video/recon_avclip/config.yaml

@@ -0,0 +1,364 @@
+seed: 1234
+audio_sample_rate: 48000
+num_frames: 193
+resolution: 256
+target_fps: 24
+video_frame_sample_rate: 1
+model:
+  video:
+    model_name: WanVAE22
+    model_config: scripts/causalvae/wanvae22_48dim.json
+    pretrained_model_name_or_path: null
+    qk_norm: true
+  audio:
+    audio_sample_rate: 48000
+    sample_rate: 48000
+    encoder_dim: 128
+    encoder_rates:
+    - 2
+    - 3
+    - 4
+    - 5
+    - 8
+    latent_dim: 128
+    decoder_dim: 2048
+    decoder_rates:
+    - 8
+    - 5
+    - 4
+    - 3
+    - 2
+    n_codebooks: 9
+    codebook_size: 1024
+    codebook_dim: 8
+    quantizer_dropout: false
+    continuous: true
+    pretrained_model_name_or_path: null
+  contrastive:
+    contrastive_type: latent_seg
+    contrastive_use_mean: true
+    contrastive_grad_scale_video: 1.0
+    contrastive_grad_scale_audio: 1.0
+    embed_dim: 512
+    segment_hidden_dim: 1024
+    global_hidden_dim: 1024
+    segment_count: null
+    init_scale: 0.07
+    clamp_scale_min: 0.001
+    clamp_scale_max: 0.5
+    gather_for_loss: true
+    num_negatives: 96
+    num_negative_videos: 63
+    same_long_video_priority: true
+    same_long_video_num_negatives: 48
+    num_negatives_with_sibling: null
+    use_sdpa: true
+    spatial_pool_mode: transformer
+    spatial_merge_factor: 2
+    transformer_dim: 768
+    spatial_transformer_layers: 1
+    segment_transformer_layers: 1
+    global_transformer_layers: 1
+    segment_temporal_pool_mode: conv
+    global_temporal_pool_mode: transformer
+    spatial_module_size: large
+    segment_module_size: large
+    global_module_size: large
+    num_negatives_no_sibling: 24
+    use_segment_loss: true
+    use_global_loss: false
+    enabled: true
+    qk_norm: true
+  distill:
+    enabled: false
+    video_target_dim: 1152
+    image_target_dim: 1152
+    audio_target_dim: 1280
+    video_spatial_factor: 4
+    image_spatial_factor: 4
+    temporal_agg_ratio: 3
+    audio_distill_type: d_axis
+    distill_proj_type: conv
+    distill_use_conv3d: false
+    distill_proj_layers: 2
+    distill_proj_hidden_dim: 256
+    distill_proj_before_agg: true
+    distill_dim_schedule: fixed
+    distill_use_sampled: false
+    audio_proj_type: conv
+    audio_proj_layers: 2
+    audio_proj_hidden_dim: 512
+  llm:
+    enabled: false
+    llm_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+    freeze_llm: false
+    include_first_frame: false
+    spatial_merge_factor: 4
+    temporal_merge_factor: 1
+    patch_embed_dim: 256
+    patch_stride: 2
+    spatial_merge_size: 2
+    audio_ds_hidden: 256
+    audio_ds_stride: 2
+    audio_inter_dim: 512
+data:
+  train:
+    metadata_paths:
+      train_main: omitted_public_release_path
+    metadata_weights:
+      train_main: 1.0
+    use_file_processor: true
+    num_frames: 193
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    num_workers: 4
+    prefetch_factor: 2
+    use_image_video_alter: false
+    image_loader: jsonl
+    image_data_mixture_yaml: null
+    image_dataset_path: null
+    image_relaion: null
+    image_batch_size: null
+    video_batch_size: null
+    image_video_weights:
+    - 1
+    - 1
+  val_video:
+    video_dir: omitted_public_release_path
+    num_frames: 193
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    max_samples: 1000
+    batch_size: 1
+  val_audio:
+    audio_dir: omitted_public_release_path
+    audio_sample_rate: 48000
+    max_duration: 5.0
+    max_samples: 1000
+    batch_size: 1
+  val_contrastive:
+    jsonl_paths:
+      vggsound_valid: data/train/vggsound_valid_filtered_original.jsonl
+      ct_nonspeech: data/train/nonspeech_valid_10000_original.jsonl
+      ct_speech: data/train/speech_valid_10000_orig.jsonl
+    num_frames: 193
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    max_samples: 20000
+    batch_size: 1
+    val_segment_num_negatives:
+    - 29
+    - 64
+    val_segment_num_negative_videos: ${model.contrastive.num_negative_videos}
+    val_global_num_negatives:
+    - 64
+    - 256
+training:
+  max_steps: 200000
+  batch_size: 1
+  lr: 0.0001
+  weight_decay: 0.0001
+  betas:
+  - 0.9
+  - 0.999
+  lr_video_vae: 2.0e-05
+  lr_audio_vae: 1.0e-05
+  lr_contrastive_head: null
+  lr_llm_caption_head: null
+  lr_distill_proj: null
+  lr_video_logvar: null
+  max_grad_norm: 1.0
+  gradient_accumulation_steps: 1
+  dtype: fp32
+  video_vae_dtype: bf16
+  audio_vae_dtype: fp32
+  contrastive_dtype: bfloat16
+  llm_dtype: bfloat16
+  lr_disc: 0.0001
+  disc_betas:
+  - 0.9
+  - 0.99
+  disc_weight_decay: 0.0
+  disc_max_grad_norm: null
+  disc_dtype: fp32
+  qk_norm: true
+  warmup_steps: 4000
+  lr_video_vae_warmup_steps: 6000
+  lr_video_vae_total_steps: null
+  lr_video_vae_start_step: null
+  lr_video_vae_min_ratio: 0.1
+  lr_audio_vae_warmup_steps: null
+  lr_audio_vae_total_steps: null
+  lr_audio_vae_start_step: null
+  lr_audio_vae_min_ratio: null
+  gradient_checkpointing: true
+  random_start: false
+  use_ema: false
+  ema_decay: 0.999
+  log_steps: 3
+  stdout_steps: 10
+  eval_steps: 2000
+  save_steps: 2000
+  max_checkpoints: 5000
+  reset_scheduler_on_resume: true
+loss:
+  use_video_recon: true
+  use_audio_recon: true
+  use_segment_contrastive: true
+  use_global_contrastive: false
+  lambda_group_video: 1.0
+  lambda_group_audio: 1.0
+  lambda_group_contrastive: 1.0
+  lambda_video_recon: 1.0
+  lambda_video_kl: 0.0001
+  lambda_video_lpips: 1.0
+  video_loss_reduction: sum
+  lambda_audio_mel: 1.0
+  lambda_audio_recon: 1.0
+  lambda_audio_kl: 0.5
+  use_audio_disc: false
+  audio_disc_start_step: 0
+  lambda_audio_adv: 1.0
+  lambda_audio_feature_matching: 1.0
+  audio_disc_params:
+    mpd: {}
+    msd: {}
+    mstftd:
+      filters: 32
+      n_ffts:
+      - 1024
+      - 2048
+      - 512
+      - 256
+      - 128
+      hop_lengths:
+      - 256
+      - 512
+      - 128
+      - 64
+      - 32
+      win_lengths:
+      - 1024
+      - 2048
+      - 512
+      - 256
+      - 128
+  use_video_disc: false
+  video_disc_start_step: 0
+  lambda_video_adv: 0.5
+  video_disc_loss_type: hinge
+  video_disc_adaptive_weight: false
+  distill_every_steps: false
+  video_disc_params:
+    input_nc: 3
+    ndf: 64
+    n_layers: 3
+    use_actnorm: false
+  lambda_segment_contrastive: 1.0
+  lambda_global_contrastive: 1.0
+  global_contrastive_start_steps: 0
+  video_loss_clamp: false
+  video_recon_clamp_max: 8000.0
+  video_lpips_clamp_max: 0.25
+  video_kl_clamp_max: 6000.0
+  adaptive_loss_balance: false
+  adaptive_balance_audio_ratio: 0.5
+  adaptive_balance_contrastive_ratio: 0.5
+  adaptive_loss_balance_by_uncertainty: false
+  uncertainty_warmup_steps: 100
+  adaptive_loss_balance_v2: true
+  adaptive_anchor_source: contrastive
+  adaptive_anchor_ema_decay: 0.99
+  adaptive_anchor_warmup_steps: 200
+  adaptive_scale_clamp_min: 0.001
+  adaptive_scale_clamp_max: 100.0
+  adaptive_ratio_video: 1.0
+  adaptive_ratio_audio: 1.0
+  adaptive_ratio_contrastive: 1.5
+  adaptive_anchor_source_stage1: contrastive
+  adaptive_ratio_video_stage1: 0.0
+  adaptive_ratio_audio_stage1: 1.0
+  adaptive_ratio_contrastive_stage1: 2.0
+  adaptive_v2_stage2_use_gradient: false
+  adaptive_v2_stage2_blend_steps: 0
+  gradient_ratio_video_stage2: null
+  gradient_ratio_audio_stage2: null
+  freeze_video_vae: true
+  freeze_video_vae_until_step: 20000
+  freeze_audio_vae: false
+  freeze_audio_vae_until_step: 0
+  use_semantic_distill: false
+  encoder_fps: 4
+  encoder_resolution: 256
+  distill_vision_layer: 18
+  distill_audio_layer: 24
+  lambda_distill_image_cosine: 1.0
+  lambda_distill_image_distance: 1.0
+  lambda_distill_video_cosine: 1.0
+  lambda_distill_video_distance: 1.0
+  lambda_distill_audio_d_axis: 120.0
+  lambda_distill_audio_t_axis: 1.0
+  distill_w_hyper: 0.1
+  distill_audio_type: d_axis
+  lambda_group_distill: 1.0
+  distill_spatial_norm: true
+  distill_spatial_norm_gamma: 0.6
+  distill_margin_cosine: 0.0
+  distill_margin_distance: 0.25
+  distill_use_dist_matrix: false
+  adaptive_distill_balance: false
+  adaptive_distill_video_ratio: 0.1
+  adaptive_distill_audio_ratio: 0.1
+  use_llm_caption: false
+  lambda_group_llm: 1.0
+  lambda_llm_caption: 1.0
+  mel_loss_kwargs:
+    sample_rate: 48000
+    n_mels:
+    - 5
+    - 10
+    - 20
+    - 40
+    - 80
+    - 160
+    - 320
+    window_lengths:
+    - 32
+    - 64
+    - 128
+    - 256
+    - 512
+    - 1024
+    - 2048
+    weights:
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    pow: 1.0
+    mel_fmin:
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    mel_fmax:
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+output:
+  exp_root: outputs

models/vae/audio_video/recon_avclip/metadata.json

@@ -0,0 +1,8 @@
+{
+  "package_type": "vae",
+  "modality": "audio_video",
+  "name": "recon_avclip",
+  "config": "models/vae/audio_video/recon_avclip/config.yaml",
+  "checkpoint": "models/vae/audio_video/recon_avclip/state_dict.pt",
+  "notes": "Unified audio-video VAE checkpoint without training state."
+}

models/vae/audio_video/recon_distill_avclip/config.yaml

@@ -0,0 +1,367 @@
+seed: 1234
+audio_sample_rate: 48000
+num_frames: 193
+resolution: 256
+target_fps: 24
+video_frame_sample_rate: 1
+model:
+  video:
+    model_name: WanVAE22
+    model_config: scripts/causalvae/wanvae22_48dim.json
+    pretrained_model_name_or_path: null
+    qk_norm: true
+  audio:
+    audio_sample_rate: 48000
+    sample_rate: 48000
+    encoder_dim: 128
+    encoder_rates:
+    - 2
+    - 3
+    - 4
+    - 5
+    - 8
+    latent_dim: 128
+    decoder_dim: 2048
+    decoder_rates:
+    - 8
+    - 5
+    - 4
+    - 3
+    - 2
+    n_codebooks: 9
+    codebook_size: 1024
+    codebook_dim: 8
+    quantizer_dropout: false
+    continuous: true
+    pretrained_model_name_or_path: null
+  contrastive:
+    contrastive_type: latent_seg
+    contrastive_use_mean: true
+    contrastive_grad_scale_video: 1.0
+    contrastive_grad_scale_audio: 1.0
+    embed_dim: 512
+    segment_hidden_dim: 1024
+    global_hidden_dim: 1024
+    segment_count: null
+    init_scale: 0.07
+    clamp_scale_min: 0.001
+    clamp_scale_max: 0.5
+    gather_for_loss: true
+    num_negatives: 96
+    num_negative_videos: 63
+    same_long_video_priority: true
+    same_long_video_num_negatives: 48
+    num_negatives_with_sibling: null
+    use_sdpa: true
+    spatial_pool_mode: transformer
+    spatial_merge_factor: 2
+    transformer_dim: 768
+    spatial_transformer_layers: 1
+    segment_transformer_layers: 1
+    global_transformer_layers: 1
+    segment_temporal_pool_mode: conv
+    global_temporal_pool_mode: transformer
+    spatial_module_size: large
+    segment_module_size: large
+    global_module_size: large
+    num_negatives_no_sibling: 24
+    use_segment_loss: true
+    use_global_loss: false
+    enabled: true
+    qk_norm: true
+  distill:
+    enabled: true
+    video_target_dim: 1152
+    image_target_dim: 1152
+    audio_target_dim: 1280
+    video_spatial_factor: 4
+    image_spatial_factor: 4
+    temporal_agg_ratio: 3
+    audio_distill_type: t_axis
+    distill_proj_type: conv
+    distill_use_conv3d: false
+    distill_proj_layers: 2
+    distill_proj_hidden_dim: 256
+    distill_proj_before_agg: true
+    distill_dim_schedule: doubling
+    distill_use_sampled: false
+    audio_proj_type: conv
+    audio_proj_layers: 2
+    audio_proj_hidden_dim: 512
+  llm:
+    enabled: false
+    llm_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+    freeze_llm: false
+    include_first_frame: false
+    spatial_merge_factor: 4
+    temporal_merge_factor: 1
+    patch_embed_dim: 256
+    patch_stride: 2
+    spatial_merge_size: 2
+    audio_ds_hidden: 256
+    audio_ds_stride: 2
+    audio_inter_dim: 512
+data:
+  train:
+    metadata_paths:
+      train_main: omitted_public_release_path
+    metadata_weights:
+      train_main: 1.0
+    use_file_processor: true
+    num_frames: 193
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    num_workers: 4
+    prefetch_factor: 2
+    use_image_video_alter: false
+    image_loader: jsonl
+    image_data_mixture_yaml: null
+    image_dataset_path: null
+    image_relaion: null
+    image_batch_size: null
+    video_batch_size: null
+    image_video_weights:
+    - 1
+    - 1
+  val_video:
+    video_dir: omitted_public_release_path
+    num_frames: 193
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    max_samples: 1000
+    batch_size: 1
+  val_audio:
+    audio_dir: omitted_public_release_path
+    audio_sample_rate: 48000
+    max_duration: 5.0
+    max_samples: 1000
+    batch_size: 1
+  val_contrastive:
+    jsonl_paths:
+      vggsound_valid: data/train/vggsound_valid_filtered_original.jsonl
+      ct_nonspeech: data/train/nonspeech_valid_10000_original.jsonl
+      ct_speech: data/train/speech_valid_10000_orig.jsonl
+    num_frames: 193
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    max_samples: 20000
+    batch_size: 1
+    val_segment_num_negatives:
+    - 29
+    - 64
+    val_segment_num_negative_videos: ${model.contrastive.num_negative_videos}
+    val_global_num_negatives:
+    - 64
+    - 256
+training:
+  max_steps: 200000
+  batch_size: 1
+  lr: 0.0001
+  weight_decay: 0.0001
+  betas:
+  - 0.9
+  - 0.999
+  lr_video_vae: 2.0e-05
+  lr_audio_vae: 5.0e-05
+  lr_contrastive_head: null
+  lr_llm_caption_head: null
+  lr_distill_proj: null
+  lr_video_logvar: null
+  max_grad_norm: 1.0
+  gradient_accumulation_steps: 1
+  dtype: fp32
+  video_vae_dtype: bf16
+  audio_vae_dtype: fp32
+  contrastive_dtype: bfloat16
+  llm_dtype: bfloat16
+  lr_disc: 0.0001
+  disc_betas:
+  - 0.9
+  - 0.99
+  disc_weight_decay: 0.0
+  disc_max_grad_norm: null
+  disc_dtype: fp32
+  qk_norm: true
+  warmup_steps: 4000
+  lr_video_vae_warmup_steps: 6000
+  lr_video_vae_total_steps: null
+  lr_video_vae_start_step: null
+  lr_video_vae_min_ratio: 0.1
+  lr_audio_vae_warmup_steps: null
+  lr_audio_vae_total_steps: null
+  lr_audio_vae_start_step: null
+  lr_audio_vae_min_ratio: null
+  gradient_checkpointing: true
+  random_start: false
+  use_ema: false
+  ema_decay: 0.999
+  log_steps: 3
+  stdout_steps: 10
+  eval_steps: 2000
+  save_steps: 2000
+  max_checkpoints: 5000
+  reset_scheduler_on_resume: true
+loss:
+  use_video_recon: true
+  use_audio_recon: true
+  use_segment_contrastive: true
+  use_global_contrastive: false
+  lambda_group_video: 1.0
+  lambda_group_audio: 1.0
+  lambda_group_contrastive: 1.0
+  lambda_video_recon: 1.0
+  lambda_video_kl: 0.0001
+  lambda_video_lpips: 1.0
+  video_loss_reduction: sum
+  lambda_audio_mel: 1.0
+  lambda_audio_recon: 1.0
+  lambda_audio_kl: 0.5
+  use_audio_disc: false
+  audio_disc_start_step: 0
+  lambda_audio_adv: 1.0
+  lambda_audio_feature_matching: 1.0
+  audio_disc_params:
+    mpd: {}
+    msd: {}
+    mstftd:
+      filters: 32
+      n_ffts:
+      - 1024
+      - 2048
+      - 512
+      - 256
+      - 128
+      hop_lengths:
+      - 256
+      - 512
+      - 128
+      - 64
+      - 32
+      win_lengths:
+      - 1024
+      - 2048
+      - 512
+      - 256
+      - 128
+  use_video_disc: false
+  video_disc_start_step: 0
+  lambda_video_adv: 0.5
+  video_disc_loss_type: hinge
+  video_disc_adaptive_weight: false
+  distill_every_steps: false
+  video_disc_params:
+    input_nc: 3
+    ndf: 64
+    n_layers: 3
+    use_actnorm: false
+  lambda_segment_contrastive: 1.0
+  lambda_global_contrastive: 1.0
+  global_contrastive_start_steps: 0
+  video_loss_clamp: false
+  video_recon_clamp_max: 8000.0
+  video_lpips_clamp_max: 0.25
+  video_kl_clamp_max: 6000.0
+  adaptive_loss_balance: false
+  adaptive_balance_audio_ratio: 0.5
+  adaptive_balance_contrastive_ratio: 0.5
+  adaptive_loss_balance_by_uncertainty: false
+  uncertainty_warmup_steps: 100
+  adaptive_loss_balance_v2: true
+  adaptive_anchor_source: contrastive
+  adaptive_anchor_ema_decay: 0.99
+  adaptive_anchor_warmup_steps: 200
+  adaptive_scale_clamp_min: 0.001
+  adaptive_scale_clamp_max: 100.0
+  adaptive_ratio_video: 1.0
+  adaptive_ratio_audio: 1.0
+  adaptive_ratio_contrastive: 1.5
+  adaptive_anchor_source_stage1: contrastive
+  adaptive_ratio_video_stage1: 0.0
+  adaptive_ratio_audio_stage1: 1.0
+  adaptive_ratio_contrastive_stage1: 2.0
+  adaptive_v2_stage2_use_gradient: false
+  adaptive_v2_stage2_blend_steps: 0
+  gradient_ratio_video_stage2: null
+  gradient_ratio_audio_stage2: null
+  freeze_video_vae: true
+  freeze_video_vae_until_step: 20000
+  freeze_audio_vae: false
+  freeze_audio_vae_until_step: 0
+  use_semantic_distill: true
+  encoder_fps: 4.0
+  encoder_resolution: 256
+  distill_vision_layer: 18
+  distill_audio_layer: 24
+  lambda_distill_image_cosine: 1.0
+  lambda_distill_image_distance: 1.0
+  lambda_distill_video_cosine: 1.0
+  lambda_distill_video_distance: 1.0
+  lambda_distill_audio_d_axis: 120.0
+  lambda_distill_audio_t_axis: 1.0
+  distill_w_hyper: 0.1
+  distill_audio_type: t_axis
+  lambda_group_distill: 1.0
+  distill_spatial_norm: true
+  distill_spatial_norm_gamma: 0.7
+  distill_margin_cosine: 0.0
+  distill_margin_distance: 0.25
+  distill_use_dist_matrix: false
+  adaptive_distill_balance: true
+  adaptive_distill_video_ratio: 0.2
+  adaptive_distill_audio_ratio: 0.2
+  use_llm_caption: false
+  lambda_group_llm: 1.0
+  lambda_llm_caption: 1.0
+  mel_loss_kwargs:
+    sample_rate: 48000
+    n_mels:
+    - 5
+    - 10
+    - 20
+    - 40
+    - 80
+    - 160
+    - 320
+    window_lengths:
+    - 32
+    - 64
+    - 128
+    - 256
+    - 512
+    - 1024
+    - 2048
+    weights:
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    pow: 1.0
+    mel_fmin:
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    mel_fmax:
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+  video_distill_start_step: 20000
+  audio_distill_start_step: 0
+  semantic_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+output:
+  exp_root: outputs

models/vae/audio_video/recon_distill_avclip/metadata.json

@@ -0,0 +1,8 @@
+{
+  "package_type": "vae",
+  "modality": "audio_video",
+  "name": "recon_distill_avclip",
+  "config": "models/vae/audio_video/recon_distill_avclip/config.yaml",
+  "checkpoint": "models/vae/audio_video/recon_distill_avclip/state_dict.pt",
+  "notes": "Unified distilled audio-video VAE checkpoint without training state."
+}

models/vae/manifest.json

@@ -0,0 +1,66 @@
+[
+  {
+    "modality": "video_only",
+    "name": "recon",
+    "path": "models/vae/video_only/recon",
+    "relative_path": "models/vae/video_only/recon",
+    "config": "models/vae/video_only/recon/config.yaml",
+    "checkpoint": "models/vae/video_only/recon/state_dict.pt"
+  },
+  {
+    "modality": "video_only",
+    "name": "recon_distill",
+    "path": "models/vae/video_only/recon_distill",
+    "relative_path": "models/vae/video_only/recon_distill",
+    "config": "models/vae/video_only/recon_distill/config.yaml",
+    "checkpoint": "models/vae/video_only/recon_distill/state_dict.pt"
+  },
+  {
+    "modality": "audio_only",
+    "name": "recon",
+    "path": "models/vae/audio_only/recon",
+    "relative_path": "models/vae/audio_only/recon",
+    "config": "models/vae/audio_only/recon/config.yaml",
+    "checkpoint": "models/vae/audio_only/recon/state_dict.pt"
+  },
+  {
+    "modality": "audio_only",
+    "name": "recon_distill",
+    "path": "models/vae/audio_only/recon_distill",
+    "relative_path": "models/vae/audio_only/recon_distill",
+    "config": "models/vae/audio_only/recon_distill/config.yaml",
+    "checkpoint": "models/vae/audio_only/recon_distill/state_dict.pt"
+  },
+  {
+    "modality": "audio_only",
+    "name": "recon_avclip_ft_decoder",
+    "path": "models/vae/audio_only/recon_avclip_ft_decoder",
+    "relative_path": "models/vae/audio_only/recon_avclip_ft_decoder",
+    "config": "models/vae/audio_only/recon_avclip_ft_decoder/config.yaml",
+    "checkpoint": "models/vae/audio_only/recon_avclip_ft_decoder/state_dict.pt"
+  },
+  {
+    "modality": "audio_only",
+    "name": "recon_distill_avclip_ft_decoder",
+    "path": "models/vae/audio_only/recon_distill_avclip_ft_decoder",
+    "relative_path": "models/vae/audio_only/recon_distill_avclip_ft_decoder",
+    "config": "models/vae/audio_only/recon_distill_avclip_ft_decoder/config.yaml",
+    "checkpoint": "models/vae/audio_only/recon_distill_avclip_ft_decoder/state_dict.pt"
+  },
+  {
+    "modality": "audio_video",
+    "name": "recon_avclip",
+    "path": "models/vae/audio_video/recon_avclip",
+    "relative_path": "models/vae/audio_video/recon_avclip",
+    "config": "models/vae/audio_video/recon_avclip/config.yaml",
+    "checkpoint": "models/vae/audio_video/recon_avclip/state_dict.pt"
+  },
+  {
+    "modality": "audio_video",
+    "name": "recon_distill_avclip",
+    "path": "models/vae/audio_video/recon_distill_avclip",
+    "relative_path": "models/vae/audio_video/recon_distill_avclip",
+    "config": "models/vae/audio_video/recon_distill_avclip/config.yaml",
+    "checkpoint": "models/vae/audio_video/recon_distill_avclip/state_dict.pt"
+  }
+]

models/vae/video_only/recon/config.yaml

@@ -0,0 +1,260 @@
+seed: 1234
+audio_sample_rate: 48000
+num_frames: 121
+resolution: 256
+target_fps: 24
+video_frame_sample_rate: 1
+model:
+  video:
+    model_name: WanVAE22
+    model_config: scripts/causalvae/wanvae22_48dim.json
+    pretrained_model_name_or_path: null
+  audio:
+    audio_sample_rate: 48000
+    sample_rate: 48000
+    encoder_dim: 128
+    encoder_rates:
+    - 2
+    - 3
+    - 4
+    - 5
+    - 8
+    latent_dim: 128
+    decoder_dim: 2048
+    decoder_rates:
+    - 8
+    - 5
+    - 4
+    - 3
+    - 2
+    n_codebooks: 9
+    codebook_size: 1024
+    codebook_dim: 8
+    quantizer_dropout: false
+    continuous: true
+    pretrained_model_name_or_path: null
+  contrastive:
+    contrastive_use_mean: true
+    embed_dim: 512
+    segment_hidden_dim: 1024
+    global_hidden_dim: 1024
+    segment_count: null
+    init_scale: 0.07
+    clamp_scale_min: 0.001
+    clamp_scale_max: 0.5
+    gather_for_loss: true
+    num_negatives: 23
+    num_negative_videos: 63
+    use_sdpa: true
+    spatial_pool_mode: mean
+    spatial_merge_factor: 4
+    transformer_dim: 768
+    spatial_transformer_layers: 1
+    segment_transformer_layers: 1
+    global_transformer_layers: 1
+    use_segment_loss: false
+    use_global_loss: false
+    enabled: false
+  distill:
+    enabled: false
+    video_target_dim: 1152
+    image_target_dim: 1152
+    audio_target_dim: 1280
+    video_spatial_factor: 4
+    image_spatial_factor: 4
+    temporal_agg_ratio: 3
+    audio_distill_type: d_axis
+    distill_proj_type: conv
+    distill_use_conv3d: false
+    distill_proj_layers: 2
+    distill_proj_hidden_dim: 256
+    distill_proj_before_agg: true
+    distill_dim_schedule: fixed
+    distill_use_sampled: false
+    audio_proj_type: conv
+    audio_proj_layers: 2
+    audio_proj_hidden_dim: 512
+  llm:
+    enabled: false
+    llm_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+    freeze_llm: false
+    include_first_frame: false
+    spatial_merge_factor: 4
+    temporal_merge_factor: 1
+    patch_embed_dim: 256
+    patch_stride: 2
+    spatial_merge_size: 2
+    audio_ds_hidden: 256
+    audio_ds_stride: 2
+    audio_inter_dim: 512
+data:
+  train:
+    metadata_paths:
+      train_main: omitted_public_release_path
+    metadata_weights:
+      train_main: 1.0
+    use_file_processor: true
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    num_workers: 4
+    prefetch_factor: 2
+  val_video:
+    video_dir: omitted_public_release_path
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    max_samples: 1000
+    batch_size: 1
+  val_audio:
+    audio_dir: omitted_public_release_path
+    audio_sample_rate: 48000
+    max_duration: 5.0
+    max_samples: 1000
+    batch_size: 1
+  val_contrastive:
+    jsonl_paths:
+      vggsound_valid: data/train/vggsound_valid_filtered.jsonl
+      ct_nonspeech: data/train/nonspeech_valid_10000.jsonl
+      ct_speech: data/train/speech_valid_10000.jsonl
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    max_samples: 20000
+    batch_size: 1
+    val_segment_num_negatives:
+    - 29
+    - 64
+    val_segment_num_negative_videos: ${model.contrastive.num_negative_videos}
+    val_global_num_negatives:
+    - 64
+    - 256
+training:
+  max_steps: 500000
+  batch_size: 1
+  lr: 1.0e-05
+  weight_decay: 0.0001
+  betas:
+  - 0.9
+  - 0.999
+  max_grad_norm: 1.0
+  gradient_accumulation_steps: 1
+  dtype: fp32
+  video_vae_dtype: bf16
+  audio_vae_dtype: fp32
+  contrastive_dtype: bfloat16
+  llm_dtype: bfloat16
+  gradient_checkpointing: true
+  random_start: false
+  use_ema: false
+  ema_decay: 0.999
+  qk_norm: false
+  log_steps: 3
+  stdout_steps: 10
+  eval_steps: 2000
+  save_steps: 2000
+  max_checkpoints: 5000
+  lr_disc: 5.0e-06
+loss:
+  use_video_recon: true
+  use_audio_recon: false
+  use_segment_contrastive: false
+  use_global_contrastive: false
+  lambda_group_video: 1.0
+  lambda_group_audio: 1.0
+  lambda_group_contrastive: 1.0
+  lambda_video_recon: 1.0
+  lambda_video_kl: 0.0001
+  lambda_video_lpips: 1.0
+  video_loss_reduction: sum
+  lambda_audio_mel: 1.0
+  lambda_audio_recon: 0.0
+  lambda_audio_kl: 0.5
+  lambda_segment_contrastive: 1.0
+  lambda_global_contrastive: 1.0
+  global_contrastive_start_steps: 0
+  adaptive_loss_balance: true
+  adaptive_balance_audio_ratio: 0.5
+  adaptive_balance_contrastive_ratio: 0.5
+  adaptive_loss_balance_by_uncertainty: false
+  uncertainty_warmup_steps: 100
+  use_semantic_distill: false
+  encoder_fps: 4
+  encoder_resolution: 256
+  distill_vision_layer: 18
+  distill_audio_layer: 24
+  lambda_distill_image_cosine: 1.0
+  lambda_distill_image_distance: 1.0
+  lambda_distill_video_cosine: 1.0
+  lambda_distill_video_distance: 1.0
+  lambda_distill_audio_d_axis: 120.0
+  lambda_distill_audio_t_axis: 1.0
+  distill_w_hyper: 0.1
+  distill_audio_type: d_axis
+  lambda_group_distill: 1.0
+  distill_spatial_norm: true
+  distill_spatial_norm_gamma: 0.6
+  distill_margin_cosine: 0.0
+  distill_margin_distance: 0.25
+  distill_use_dist_matrix: false
+  adaptive_distill_balance: false
+  adaptive_distill_video_ratio: 0.1
+  adaptive_distill_audio_ratio: 0.1
+  use_llm_caption: false
+  lambda_group_llm: 1.0
+  lambda_llm_caption: 1.0
+  mel_loss_kwargs:
+    sample_rate: 48000
+    n_mels:
+    - 5
+    - 10
+    - 20
+    - 40
+    - 80
+    - 160
+    - 320
+    window_lengths:
+    - 32
+    - 64
+    - 128
+    - 256
+    - 512
+    - 1024
+    - 2048
+    weights:
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    pow: 1.0
+    mel_fmin:
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    mel_fmax:
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+  use_video_disc: true
+  video_disc_start_step: 3000
+  lambda_video_adv: 0.1
+  video_disc_loss_type: hinge
+  video_disc_adaptive_weight: true
+output:
+  exp_root: outputs

models/vae/video_only/recon/metadata.json

@@ -0,0 +1,8 @@
+{
+  "package_type": "vae",
+  "modality": "video_only",
+  "name": "recon",
+  "config": "models/vae/video_only/recon/config.yaml",
+  "checkpoint": "models/vae/video_only/recon/state_dict.pt",
+  "notes": "Video reconstruction VAE checkpoint without training state."
+}

models/vae/video_only/recon_distill/config.yaml

@@ -0,0 +1,262 @@
+seed: 1234
+audio_sample_rate: 48000
+num_frames: 121
+resolution: 256
+target_fps: 24
+video_frame_sample_rate: 1
+model:
+  video:
+    model_name: WanVAE22
+    model_config: scripts/causalvae/wanvae22_48dim.json
+    pretrained_model_name_or_path: null
+  audio:
+    audio_sample_rate: 48000
+    sample_rate: 48000
+    encoder_dim: 128
+    encoder_rates:
+    - 2
+    - 3
+    - 4
+    - 5
+    - 8
+    latent_dim: 128
+    decoder_dim: 2048
+    decoder_rates:
+    - 8
+    - 5
+    - 4
+    - 3
+    - 2
+    n_codebooks: 9
+    codebook_size: 1024
+    codebook_dim: 8
+    quantizer_dropout: false
+    continuous: true
+    pretrained_model_name_or_path: null
+  contrastive:
+    contrastive_use_mean: true
+    embed_dim: 512
+    segment_hidden_dim: 1024
+    global_hidden_dim: 1024
+    segment_count: null
+    init_scale: 0.07
+    clamp_scale_min: 0.001
+    clamp_scale_max: 0.5
+    gather_for_loss: true
+    num_negatives: 23
+    num_negative_videos: 63
+    use_sdpa: true
+    spatial_pool_mode: mean
+    spatial_merge_factor: 4
+    transformer_dim: 768
+    spatial_transformer_layers: 1
+    segment_transformer_layers: 1
+    global_transformer_layers: 1
+    use_segment_loss: false
+    use_global_loss: false
+    enabled: false
+  distill:
+    enabled: true
+    video_target_dim: 1152
+    image_target_dim: 1152
+    audio_target_dim: 1280
+    video_spatial_factor: 4
+    image_spatial_factor: 4
+    temporal_agg_ratio: 3
+    audio_distill_type: d_axis
+    distill_proj_type: conv
+    distill_use_conv3d: false
+    distill_proj_layers: 2
+    distill_proj_hidden_dim: 256
+    distill_proj_before_agg: true
+    distill_dim_schedule: doubling
+    distill_use_sampled: false
+    audio_proj_type: conv
+    audio_proj_layers: 2
+    audio_proj_hidden_dim: 512
+  llm:
+    enabled: false
+    llm_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+    freeze_llm: false
+    include_first_frame: false
+    spatial_merge_factor: 4
+    temporal_merge_factor: 1
+    patch_embed_dim: 256
+    patch_stride: 2
+    spatial_merge_size: 2
+    audio_ds_hidden: 256
+    audio_ds_stride: 2
+    audio_inter_dim: 512
+data:
+  train:
+    metadata_paths:
+      train_main: omitted_public_release_path
+    metadata_weights:
+      train_main: 1.0
+    use_file_processor: true
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    num_workers: 4
+    prefetch_factor: 2
+  val_video:
+    video_dir: omitted_public_release_path
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    max_samples: 1000
+    batch_size: 1
+  val_audio:
+    audio_dir: omitted_public_release_path
+    audio_sample_rate: 48000
+    max_duration: 5.0
+    max_samples: 1000
+    batch_size: 1
+  val_contrastive:
+    jsonl_paths:
+      vggsound_valid: data/train/vggsound_valid_filtered.jsonl
+      ct_nonspeech: data/train/nonspeech_valid_10000.jsonl
+      ct_speech: data/train/speech_valid_10000.jsonl
+    num_frames: 121
+    resolution: 256
+    video_frame_sample_rate: 1
+    target_fps: 24
+    audio_sample_rate: 48000
+    max_samples: 20000
+    batch_size: 1
+    val_segment_num_negatives:
+    - 29
+    - 64
+    val_segment_num_negative_videos: ${model.contrastive.num_negative_videos}
+    val_global_num_negatives:
+    - 64
+    - 256
+training:
+  max_steps: 500000
+  batch_size: 1
+  lr: 1.0e-05
+  weight_decay: 0.0001
+  betas:
+  - 0.9
+  - 0.999
+  max_grad_norm: 1.0
+  gradient_accumulation_steps: 1
+  dtype: fp32
+  video_vae_dtype: bf16
+  audio_vae_dtype: fp32
+  contrastive_dtype: bfloat16
+  llm_dtype: bfloat16
+  gradient_checkpointing: true
+  random_start: false
+  use_ema: false
+  ema_decay: 0.999
+  qk_norm: false
+  log_steps: 3
+  stdout_steps: 10
+  eval_steps: 2000
+  save_steps: 2000
+  max_checkpoints: 5000
+  lr_disc: 5.0e-06
+loss:
+  use_video_recon: true
+  use_audio_recon: false
+  use_segment_contrastive: false
+  use_global_contrastive: false
+  lambda_group_video: 1.0
+  lambda_group_audio: 1.0
+  lambda_group_contrastive: 1.0
+  lambda_video_recon: 1.0
+  lambda_video_kl: 0.0001
+  lambda_video_lpips: 1.0
+  video_loss_reduction: sum
+  lambda_audio_mel: 1.0
+  lambda_audio_recon: 0.0
+  lambda_audio_kl: 0.5
+  lambda_segment_contrastive: 1.0
+  lambda_global_contrastive: 1.0
+  global_contrastive_start_steps: 0
+  adaptive_loss_balance: true
+  adaptive_balance_audio_ratio: 0.5
+  adaptive_balance_contrastive_ratio: 0.5
+  adaptive_loss_balance_by_uncertainty: false
+  uncertainty_warmup_steps: 100
+  use_semantic_distill: true
+  encoder_fps: 4.0
+  encoder_resolution: 256
+  distill_vision_layer: 18
+  distill_audio_layer: 24
+  lambda_distill_image_cosine: 1.0
+  lambda_distill_image_distance: 1.0
+  lambda_distill_video_cosine: 1.0
+  lambda_distill_video_distance: 1.0
+  lambda_distill_audio_d_axis: 120.0
+  lambda_distill_audio_t_axis: 1.0
+  distill_w_hyper: 0.1
+  distill_audio_type: d_axis
+  lambda_group_distill: 1.0
+  distill_spatial_norm: true
+  distill_spatial_norm_gamma: 0.7
+  distill_margin_cosine: 0.0
+  distill_margin_distance: 0.25
+  distill_use_dist_matrix: false
+  adaptive_distill_balance: true
+  adaptive_distill_video_ratio: 0.5
+  adaptive_distill_audio_ratio: 0.1
+  use_llm_caption: false
+  lambda_group_llm: 1.0
+  lambda_llm_caption: 1.0
+  mel_loss_kwargs:
+    sample_rate: 48000
+    n_mels:
+    - 5
+    - 10
+    - 20
+    - 40
+    - 80
+    - 160
+    - 320
+    window_lengths:
+    - 32
+    - 64
+    - 128
+    - 256
+    - 512
+    - 1024
+    - 2048
+    weights:
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    - 15
+    pow: 1.0
+    mel_fmin:
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    - 0.0
+    mel_fmax:
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+    - null
+  use_video_disc: true
+  video_disc_start_step: 3000
+  lambda_video_adv: 0.1
+  video_disc_loss_type: hinge
+  video_disc_adaptive_weight: true
+  distill_every_steps: true
+  semantic_model_path: models/text_encoder/Qwen3.5-0.8B-Base
+output:
+  exp_root: outputs

models/vae/video_only/recon_distill/metadata.json

@@ -0,0 +1,8 @@
+{
+  "package_type": "vae",
+  "modality": "video_only",
+  "name": "recon_distill",
+  "config": "models/vae/video_only/recon_distill/config.yaml",
+  "checkpoint": "models/vae/video_only/recon_distill/state_dict.pt",
+  "notes": "Distilled video reconstruction VAE checkpoint without training state."
+}