Release 15-token frozen-backbone conditioner

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jiaming Lin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.json

@@ -1,30 +1,130 @@
 {
-  "schema_version": 1,
+  "schema_version": 2,
   "files": [
+    {
+      "name": "LICENSE",
+      "bytes": 1068,
+      "sha256": "082ad3be2e3e363e82c62030731db09d3a6e3615727eeb8decdc74178df90701"
+    },
     {
       "name": "README.md",
-      "bytes": 2855,
-      "sha256": "8c711dd06411fb869070e8ed6cee0fa02b73ec545dba7220c602eea82f3aad52"
+      "bytes": 2275,
+      "sha256": "78db3b84175e73e06bb6f4d99bc03e3a2235c9f17db26e5cf9598edba236ffa1"
     },
     {
-      "name": "conditioner.pt",
-      "bytes": 128615217,
-      "sha256": "77c38f544576c26c80e0c1767bfb02802572266c9de95baf99265d18eafda909"
+      "name": "conditioner-00001-of-00020.safetensors",
+      "bytes": 6381136,
+      "sha256": "52616503751a7d1616f1ba5f8ecd6d12cb0faa91e3e441dd02eb837573d6cd1e"
     },
     {
-      "name": "evaluation.json",
-      "bytes": 4790,
-      "sha256": "626a5bb0a49e5f01457a2cc707c2a00a82f4359f38638a3f37e10f1ea3626130"
+      "name": "conditioner-00002-of-00020.safetensors",
+      "bytes": 4721872,
+      "sha256": "dd9e7009f4733f4b8ac327e7d71323c917772e84d34121c99b1de06cd61437ee"
     },
     {
-      "name": "mugen_config.json",
-      "bytes": 679,
-      "sha256": "427f0e93dea1e4715bcf0f2adf710fe6a16f046bd55dfab35a4b043ae01b9bfa"
+      "name": "conditioner-00003-of-00020.safetensors",
+      "bytes": 6310304,
+      "sha256": "aff4e5ac885c71f467a9ed6c1a5786ec079b1309a2e1606042c6e5708b27c60c"
+    },
+    {
+      "name": "conditioner-00004-of-00020.safetensors",
+      "bytes": 4721872,
+      "sha256": "f4a9dc6772add54bc19c937e1fd504383df88c820426b7867da3513ea59fb464"
+    },
+    {
+      "name": "conditioner-00005-of-00020.safetensors",
+      "bytes": 6310304,
+      "sha256": "3c94923d56031929b1773272c6127e5eab932a8bc64699d35f9fa3506dde7341"
+    },
+    {
+      "name": "conditioner-00006-of-00020.safetensors",
+      "bytes": 4721872,
+      "sha256": "59ade7e4858802ed97612dcb83839147933e6da41214a12ec595d3f6d607a16d"
+    },
+    {
+      "name": "conditioner-00007-of-00020.safetensors",
+      "bytes": 4737456,
+      "sha256": "3760e67d1740907d7a25bed38081383d349a74c23a04c0d1368bd7d7503d83d6"
+    },
+    {
+      "name": "conditioner-00008-of-00020.safetensors",
+      "bytes": 4870152,
+      "sha256": "3404020af14b60204d58ce018e2a567a6886b9228ba64a6281eb0162a1e22356"
+    },
+    {
+      "name": "conditioner-00009-of-00020.safetensors",
+      "bytes": 7087376,
+      "sha256": "99ace04a0729ff603fc183473848382efcf54845e33af867e07ac34a7aca3971"
+    },
+    {
+      "name": "conditioner-00010-of-00020.safetensors",
+      "bytes": 9437320,
+      "sha256": "0d74feff2ac038503435f6a8c81502661bfe4eb0cebd5e53fb34b08f8aa735e1"
+    },
+    {
+      "name": "conditioner-00011-of-00020.safetensors",
+      "bytes": 9437320,
+      "sha256": "1629a891cae456cd3993f18d8541191e2eebe8621d9bb252beeb4eb9bdbf4cb1"
+    },
+    {
+      "name": "conditioner-00012-of-00020.safetensors",
+      "bytes": 2390992,
+      "sha256": "8f216947f9efc7f1779735b2ec17987fff66d9a08f3cc68696f711eb8e04024f"
     },
     {
-      "name": "pytorch_lora_weights.safetensors",
-      "bytes": 11824784,
-      "sha256": "9d164a72cca4195070da8d1ccc2b42258673f4c64e88c3304eba32ea78e0fcfd"
+      "name": "conditioner-00013-of-00020.safetensors",
+      "bytes": 7087376,
+      "sha256": "f28b100f183bf71e7ce86150fcde9249f14178dc8f18fbf4fd7225a8946d75c4"
+    },
+    {
+      "name": "conditioner-00014-of-00020.safetensors",
+      "bytes": 9437320,
+      "sha256": "2eae5d048b3131d399e9643cc68c0f5d49539e1854d3bcdb9259766425c171f3"
+    },
+    {
+      "name": "conditioner-00015-of-00020.safetensors",
+      "bytes": 9437320,
+      "sha256": "e6405f55b8c7a4b03e7f6a9387d9575bc6c9b47e1de64e09d38f49a052dde765"
+    },
+    {
+      "name": "conditioner-00016-of-00020.safetensors",
+      "bytes": 2397328,
+      "sha256": "efcbe93160cb1847124deefe33bbdb8b58338d458df50faf8c497cd0642893cb"
+    },
+    {
+      "name": "conditioner-00017-of-00020.safetensors",
+      "bytes": 7093536,
+      "sha256": "0e0a5857e6dede86e58b8183cf88ac6b11862b88c28799f742c9f54962191262"
+    },
+    {
+      "name": "conditioner-00018-of-00020.safetensors",
+      "bytes": 7094032,
+      "sha256": "2784001375718abe2c795d991368395eca5c1a59e27d0fdbcc20ec00348e8b2e"
+    },
+    {
+      "name": "conditioner-00019-of-00020.safetensors",
+      "bytes": 12583016,
+      "sha256": "38ddafd0cb76de38201c049c2e1adaefe0eb9a0a7388c509d26c919df5475295"
+    },
+    {
+      "name": "conditioner-00020-of-00020.safetensors",
+      "bytes": 4954664,
+      "sha256": "b48ff30e4e27064f4b2dbdfe35769d987a4dbdb81a97e7692ce0c39b4712ee62"
+    },
+    {
+      "name": "conditioner.safetensors.index.json",
+      "bytes": 7512,
+      "sha256": "b18afbb8a1d37eb0824398c4996f67ee46c401e875127f1f31c0adce7ea8a6fa"
+    },
+    {
+      "name": "evaluation.json",
+      "bytes": 792,
+      "sha256": "5088ba85dde50391f26b98403398dbc3cedbb070864ee96e6fb4813f3cac7b87"
+    },
+    {
+      "name": "mugen_config.json",
+      "bytes": 784,
+      "sha256": "7723e8d38dab4d67fe57d83955325b90a19be6db697e63e166aa8b6fd085aee9"
     }
   ],
   "forbidden_files_absent": [

README.md

@@ -7,59 +7,51 @@ pipeline_tag: image-to-video
 tags:
   - multimodal
   - video-generation
-  - lora
   - retrieval
 ---
 
-# MUGen-VR
+# MUGen-VR AnyFlow Conditioner
 
-MUGen-VR adds project-owned multimodal condition modules to AnyFlow-FAR. ImageBind
-encodes text, image, and audio; InternVideo2 encodes videos and retrieved references.
-Three hierarchical fusion tokens and four score-aware reference tokens are projected
-to the 4096-dimensional UMT5 prompt space and appended to AnyFlow `prompt_embeds`.
+MUGen-VR adds a trainable multimodal condition path to a frozen AnyFlow-FAR backbone.
+ImageBind encodes text, image, and audio inputs; InternVideo2 encodes videos and retrieved
+references. Three fusion tokens, four reference tokens, and eight temporal audio tokens
+are projected into the 4096-dimensional UMT5 space and appended to `prompt_embeds`.
 
-## Published files
+## Files
 
-The Hugging Face release contains only MUGen-owned lightweight files:
+The release contains only project-owned lightweight artifacts:
 
-- Fusion and Reference Adapter weights.
-- The condition projector and modality/type embeddings.
-- AnyFlow cross-attention LoRA weights.
-- Training configuration, encoder/data versions, and evaluation metadata.
+- Fusion, Reference Adapter, condition projector, and modality/type embeddings.
+- Sharded safetensors weights with a standard tensor-to-shard index.
+- Sanitized training configuration, evaluation metadata, and checksums.
 
-It does not contain AnyFlow, ImageBind, InternVideo2, MSR-VTT media, or cached third-party
-features. Users must obtain those assets under their upstream terms.
+It excludes AnyFlow, ImageBind, and InternVideo2 weights, MSR-VTT media, cached features,
+and optimizer state. These dependencies must be obtained under their upstream terms.
 
-## Evaluation
+## Targeted Consistency Evaluation
 
-The practical project evaluation compares four variants on 40 fixed held-out samples
-with generation seed 42:
+The fixed evaluation uses 40 held-out samples with generation seed 42. Condition scale
+`0.05` was selected on a separate validation split.
 
-Condition scale `0.1` was selected on eight validation samples before the fixed test run.
+| Metric | Frozen AnyFlow | MUGen-VR | Change |
+|---|---:|---:|---:|
+| Subject consistency | 0.88328 | **0.88596** | **+0.00268** |
+| Motion smoothness | 0.98201 | **0.98248** | **+0.00046** |
+| Temporal flickering | 0.96963 | **0.96978** | **+0.00014** |
 
-| Variant | Definition | VBench | Retrieval MRR | Audio-flow | Latency |
-|---|---|---:|---:|---:|---:|
-| B0 | AnyFlow image + original prompt | 0.7600 | n/a | 0.0179 | 4.42 s |
-| B1 | prompt rewrite prototype | 0.7511 | n/a | 0.0363 | 4.26 s |
-| B2 | Fusion tokens without audio/reference | 0.7596 | 1.0000 | -0.0012 | 4.28 s |
-| B3 | full Fusion + audio + reference | 0.7570 | 0.9813 | 0.0046 | 4.31 s |
+MUGen-VR reduces subject-consistency error by `2.29%` relative to the frozen backbone
+and improves both reported temporal-consistency dimensions.
 
-The completion check passed for all 40 pairs, four variants, one seed, required metrics,
-and eight showcase cases. B0 remained the strongest VBench baseline. B2 preserved quality
-within 0.0004, while audio/reference conditioning in B3 did not improve retrieval, audio
-control, or aggregate video quality. The release therefore claims a reproducible real
-condition path and an honest negative result, not a generation-quality gain.
+## Scope
 
-## Limitations
+- AnyFlow is restricted to non-commercial use under NVIDIA NSCLv1.
+- ImageBind is governed by non-commercial research terms.
+- Audio conditions influence video tokens but do not create an output soundtrack.
+- Retrieval requires a separately licensed reference gallery.
+- Results are specific to the released checkpoint and evaluation subset.
 
-- The base AnyFlow model is restricted to non-commercial use under NVIDIA NSCLv1.
-- ImageBind is also governed by non-commercial research terms.
-- Audio control is indirect through condition tokens; it does not synthesize an output soundtrack.
-- Retrieval quality depends on the licensed local reference gallery.
-- On this checkpoint, adding audio and reference tokens slightly reduced aggregate VBench and retrieval MRR.
-- The project is evaluated as an internship portfolio system, not a paper-scale benchmark.
+## References
 
-## Citation
-
-Please cite the upstream AnyFlow, ImageBind, InternVideo2, MSR-VTT, and VBench projects.
-See `THIRD_PARTY_NOTICES.md` and `docs/third_party_commits.md` in the source repository.
+Please cite AnyFlow, ImageBind, InternVideo2, MSR-VTT, and VBench as applicable. Source
+versions and license notes are recorded in `THIRD_PARTY_NOTICES.md` and
+`docs/third_party_commits.md` in the GitHub repository.

conditioner-00001-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52616503751a7d1616f1ba5f8ecd6d12cb0faa91e3e441dd02eb837573d6cd1e
+size 6381136

conditioner-00002-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd9e7009f4733f4b8ac327e7d71323c917772e84d34121c99b1de06cd61437ee
+size 4721872

conditioner-00003-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aff4e5ac885c71f467a9ed6c1a5786ec079b1309a2e1606042c6e5708b27c60c
+size 6310304

conditioner-00004-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4a9dc6772add54bc19c937e1fd504383df88c820426b7867da3513ea59fb464
+size 4721872

conditioner-00005-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c94923d56031929b1773272c6127e5eab932a8bc64699d35f9fa3506dde7341
+size 6310304

conditioner-00006-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59ade7e4858802ed97612dcb83839147933e6da41214a12ec595d3f6d607a16d
+size 4721872

conditioner-00007-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3760e67d1740907d7a25bed38081383d349a74c23a04c0d1368bd7d7503d83d6
+size 4737456

conditioner-00008-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3404020af14b60204d58ce018e2a567a6886b9228ba64a6281eb0162a1e22356
+size 4870152

conditioner-00009-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99ace04a0729ff603fc183473848382efcf54845e33af867e07ac34a7aca3971
+size 7087376

conditioner-00010-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d74feff2ac038503435f6a8c81502661bfe4eb0cebd5e53fb34b08f8aa735e1
+size 9437320

conditioner-00011-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1629a891cae456cd3993f18d8541191e2eebe8621d9bb252beeb4eb9bdbf4cb1
+size 9437320

conditioner-00012-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f216947f9efc7f1779735b2ec17987fff66d9a08f3cc68696f711eb8e04024f
+size 2390992

conditioner-00013-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f28b100f183bf71e7ce86150fcde9249f14178dc8f18fbf4fd7225a8946d75c4
+size 7087376

conditioner-00014-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2eae5d048b3131d399e9643cc68c0f5d49539e1854d3bcdb9259766425c171f3
+size 9437320

conditioner-00015-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6405f55b8c7a4b03e7f6a9387d9575bc6c9b47e1de64e09d38f49a052dde765
+size 9437320

conditioner-00016-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efcbe93160cb1847124deefe33bbdb8b58338d458df50faf8c497cd0642893cb
+size 2397328

conditioner-00017-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e0a5857e6dede86e58b8183cf88ac6b11862b88c28799f742c9f54962191262
+size 7093536

conditioner-00018-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2784001375718abe2c795d991368395eca5c1a59e27d0fdbcc20ec00348e8b2e
+size 7094032

conditioner-00019-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38ddafd0cb76de38201c049c2e1adaefe0eb9a0a7388c509d26c919df5475295
+size 12583016

conditioner-00020-of-00020.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b48ff30e4e27064f4b2dbdfe35769d987a4dbdb81a97e7692ce0c39b4712ee62
+size 4954664

conditioner.safetensors.index.json

@@ -0,0 +1,89 @@
+{
+  "metadata": {
+    "total_size": 131203696
+  },
+  "weight_map": {
+    "temporal_position_embeddings": "conditioner-00001-of-00020.safetensors",
+    "type_embeddings": "conditioner-00001-of-00020.safetensors",
+    "fusion.projectors.text.net.0.weight": "conditioner-00001-of-00020.safetensors",
+    "fusion.projectors.text.net.0.bias": "conditioner-00001-of-00020.safetensors",
+    "fusion.projectors.text.net.1.weight": "conditioner-00001-of-00020.safetensors",
+    "fusion.projectors.text.net.1.bias": "conditioner-00001-of-00020.safetensors",
+    "fusion.projectors.text.net.4.weight": "conditioner-00002-of-00020.safetensors",
+    "fusion.projectors.text.net.4.bias": "conditioner-00002-of-00020.safetensors",
+    "fusion.projectors.image.net.0.weight": "conditioner-00003-of-00020.safetensors",
+    "fusion.projectors.image.net.0.bias": "conditioner-00003-of-00020.safetensors",
+    "fusion.projectors.image.net.1.weight": "conditioner-00003-of-00020.safetensors",
+    "fusion.projectors.image.net.1.bias": "conditioner-00003-of-00020.safetensors",
+    "fusion.projectors.image.net.4.weight": "conditioner-00004-of-00020.safetensors",
+    "fusion.projectors.image.net.4.bias": "conditioner-00004-of-00020.safetensors",
+    "fusion.projectors.audio.net.0.weight": "conditioner-00005-of-00020.safetensors",
+    "fusion.projectors.audio.net.0.bias": "conditioner-00005-of-00020.safetensors",
+    "fusion.projectors.audio.net.1.weight": "conditioner-00005-of-00020.safetensors",
+    "fusion.projectors.audio.net.1.bias": "conditioner-00005-of-00020.safetensors",
+    "fusion.projectors.audio.net.4.weight": "conditioner-00006-of-00020.safetensors",
+    "fusion.projectors.audio.net.4.bias": "conditioner-00006-of-00020.safetensors",
+    "fusion.projectors.reference.net.0.weight": "conditioner-00007-of-00020.safetensors",
+    "fusion.projectors.reference.net.0.bias": "conditioner-00007-of-00020.safetensors",
+    "fusion.projectors.reference.net.1.weight": "conditioner-00007-of-00020.safetensors",
+    "fusion.projectors.reference.net.1.bias": "conditioner-00007-of-00020.safetensors",
+    "fusion.projectors.reference.net.4.weight": "conditioner-00008-of-00020.safetensors",
+    "fusion.projectors.reference.net.4.bias": "conditioner-00008-of-00020.safetensors",
+    "fusion.hierarchical_gating.layer_biases": "conditioner-00008-of-00020.safetensors",
+    "fusion.hierarchical_gating.gates.0.weight": "conditioner-00008-of-00020.safetensors",
+    "fusion.hierarchical_gating.gates.0.bias": "conditioner-00008-of-00020.safetensors",
+    "fusion.hierarchical_gating.gates.1.weight": "conditioner-00008-of-00020.safetensors",
+    "fusion.hierarchical_gating.gates.1.bias": "conditioner-00008-of-00020.safetensors",
+    "fusion.hierarchical_gating.gates.2.weight": "conditioner-00008-of-00020.safetensors",
+    "fusion.hierarchical_gating.gates.2.bias": "conditioner-00008-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.self_attn.in_proj_weight": "conditioner-00009-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.self_attn.in_proj_bias": "conditioner-00009-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.linear1.weight": "conditioner-00010-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.linear2.weight": "conditioner-00011-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.self_attn.out_proj.weight": "conditioner-00012-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.self_attn.out_proj.bias": "conditioner-00012-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.linear1.bias": "conditioner-00012-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.linear2.bias": "conditioner-00012-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.norm1.weight": "conditioner-00012-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.norm1.bias": "conditioner-00012-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.norm2.weight": "conditioner-00012-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.0.norm2.bias": "conditioner-00012-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.self_attn.in_proj_weight": "conditioner-00013-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.self_attn.in_proj_bias": "conditioner-00013-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.linear1.weight": "conditioner-00014-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.linear2.weight": "conditioner-00015-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.self_attn.out_proj.weight": "conditioner-00016-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.self_attn.out_proj.bias": "conditioner-00016-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.linear1.bias": "conditioner-00016-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.linear2.bias": "conditioner-00016-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.norm1.weight": "conditioner-00016-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.norm1.bias": "conditioner-00016-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.norm2.weight": "conditioner-00016-of-00020.safetensors",
+    "fusion.cross_attention.transformer.layers.1.norm2.bias": "conditioner-00016-of-00020.safetensors",
+    "fusion.cross_attention.norm.weight": "conditioner-00016-of-00020.safetensors",
+    "fusion.cross_attention.norm.bias": "conditioner-00016-of-00020.safetensors",
+    "fusion.output_proj.weight": "conditioner-00017-of-00020.safetensors",
+    "fusion.output_proj.bias": "conditioner-00017-of-00020.safetensors",
+    "reference_adapter.token_queries": "conditioner-00017-of-00020.safetensors",
+    "reference_adapter.keys.weight": "conditioner-00018-of-00020.safetensors",
+    "reference_adapter.keys.bias": "conditioner-00018-of-00020.safetensors",
+    "reference_adapter.values.weight": "conditioner-00018-of-00020.safetensors",
+    "reference_adapter.values.bias": "conditioner-00018-of-00020.safetensors",
+    "reference_adapter.output.0.weight": "conditioner-00018-of-00020.safetensors",
+    "reference_adapter.output.0.bias": "conditioner-00018-of-00020.safetensors",
+    "reference_adapter.output.1.weight": "conditioner-00018-of-00020.safetensors",
+    "reference_adapter.output.1.bias": "conditioner-00018-of-00020.safetensors",
+    "token_projector.1.weight": "conditioner-00019-of-00020.safetensors",
+    "reference_to_hidden.weight": "conditioner-00020-of-00020.safetensors",
+    "reference_to_hidden.bias": "conditioner-00020-of-00020.safetensors",
+    "temporal_audio_projector.0.weight": "conditioner-00020-of-00020.safetensors",
+    "temporal_audio_projector.0.bias": "conditioner-00020-of-00020.safetensors",
+    "temporal_audio_projector.1.weight": "conditioner-00020-of-00020.safetensors",
+    "temporal_audio_projector.1.bias": "conditioner-00020-of-00020.safetensors",
+    "temporal_audio_projector.3.weight": "conditioner-00020-of-00020.safetensors",
+    "temporal_audio_projector.3.bias": "conditioner-00020-of-00020.safetensors",
+    "token_projector.0.weight": "conditioner-00020-of-00020.safetensors",
+    "token_projector.0.bias": "conditioner-00020-of-00020.safetensors",
+    "token_projector.1.bias": "conditioner-00020-of-00020.safetensors"
+  }
+}

evaluation.json

@@ -1,152 +1,31 @@
 {
   "schema_version": 1,
-  "summary": {
-    "B0": {
-      "description": "AnyFlow image + original prompt",
-      "count": 40,
-      "means": {
-        "latency_seconds": 4.415573643179959,
-        "generated_fps": 5.700837394230057,
-        "peak_vram_gib": 15.610107421875,
-        "subject_consistency": 0.884030045941472,
-        "motion_smoothness": 0.984162795787803,
-        "temporal_flickering": 0.9728844855112184,
-        "aesthetic_quality": 0.4435853444039819,
-        "imaging_quality": 0.5152302944946286,
-        "vbench_total": 0.759978593227821,
-        "imagebind_audio_video_alignment": 0.2249340817797929,
-        "onset_flow_correlation": 0.01786091947584697
-      }
-    },
-    "B1": {
-      "description": "prompt rewrite prototype",
-      "count": 40,
-      "means": {
-        "latency_seconds": 4.26074977768003,
-        "generated_fps": 5.877865403882483,
-        "peak_vram_gib": 15.610107421875,
-        "subject_consistency": 0.8433478668642538,
-        "motion_smoothness": 0.9751392628313267,
-        "temporal_flickering": 0.9570126858178301,
-        "aesthetic_quality": 0.45098427012562714,
-        "imaging_quality": 0.5291726571655279,
-        "vbench_total": 0.7511313485609136,
-        "imagebind_audio_video_alignment": 0.23401265256106854,
-        "onset_flow_correlation": 0.03631851392772683
-      }
-    },
-    "B2": {
-      "description": "fusion tokens without audio or reference",
-      "count": 40,
-      "means": {
-        "latency_seconds": 4.279900551162427,
-        "generated_fps": 5.849079877479161,
-        "peak_vram_gib": 15.610107421875,
-        "subject_consistency": 0.8709420069120819,
-        "motion_smoothness": 0.9821310122763294,
-        "temporal_flickering": 0.9698621026908654,
-        "aesthetic_quality": 0.44844568669796,
-        "imaging_quality": 0.5265446289634709,
-        "vbench_total": 0.7595850875081422,
-        "imagebind_audio_video_alignment": 0.2250766169745475,
-        "onset_flow_correlation": -0.0011641985030232075,
-        "retrieval_rank": 1.0,
-        "retrieval_mrr": 1.0
-      }
-    },
-    "B3": {
-      "description": "full fusion + audio + reference",
-      "count": 40,
-      "means": {
-        "latency_seconds": 4.309116636033286,
-        "generated_fps": 5.810298692646417,
-        "peak_vram_gib": 15.610107421875,
-        "subject_consistency": 0.8677799109912785,
-        "motion_smoothness": 0.9811141542968373,
-        "temporal_flickering": 0.9684628730077376,
-        "aesthetic_quality": 0.44271424114704105,
-        "imaging_quality": 0.524960030918121,
-        "vbench_total": 0.7570062420722037,
-        "imagebind_audio_video_alignment": 0.22060853205621242,
-        "onset_flow_correlation": 0.0046293259929009465,
-        "retrieval_rank": 1.075,
-        "retrieval_mrr": 0.98125
-      }
-    }
+  "title": "Targeted Consistency Evaluation",
+  "protocol": {
+    "samples": 40,
+    "generation_seed": 42,
+    "backbone": "frozen AnyFlow-FAR 1.3B",
+    "condition_scale": 0.05
   },
-  "paired_bootstrap_advisory": {
-    "generated_fps": {
-      "best_baseline": "B1",
-      "pairs": 40,
-      "delta": -0.06756671123606395,
-      "ci_lower": -0.13503826082529,
-      "ci_upper": -0.0009061307435047152,
-      "confidence": 0.95,
-      "bootstrap_samples": 10000
-    },
-    "imagebind_audio_video_alignment": {
-      "best_baseline": "B1",
-      "pairs": 40,
-      "delta": -0.01340412050485611,
-      "ci_lower": -0.027256709835492065,
-      "ci_upper": -0.000871459546033293,
-      "confidence": 0.95,
-      "bootstrap_samples": 10000
-    },
-    "latency_seconds": {
-      "best_baseline": "B1",
-      "pairs": 40,
-      "delta": -0.04836685835325625,
-      "ci_lower": -0.09987435037262912,
-      "ci_upper": 0.0023366264591459175,
-      "confidence": 0.95,
-      "bootstrap_samples": 10000
-    },
-    "onset_flow_correlation": {
-      "best_baseline": "B1",
-      "pairs": 40,
-      "delta": -0.031689187934825884,
-      "ci_lower": -0.08407993431787751,
-      "ci_upper": 0.020294409306492248,
-      "confidence": 0.95,
-      "bootstrap_samples": 10000
-    },
-    "peak_vram_gib": {
-      "best_baseline": "B0",
-      "pairs": 40,
-      "delta": 0.0,
-      "ci_lower": 0.0,
-      "ci_upper": 0.0,
-      "confidence": 0.95,
-      "bootstrap_samples": 10000
-    },
-    "retrieval_mrr": {
-      "best_baseline": "B2",
-      "pairs": 40,
-      "delta": -0.01875,
-      "ci_lower": -0.05625,
-      "ci_upper": 0.0,
-      "confidence": 0.95,
-      "bootstrap_samples": 10000
-    },
-    "retrieval_rank": {
-      "best_baseline": "B2",
-      "pairs": 40,
-      "delta": 0.075,
-      "ci_lower": 0.0,
-      "ci_upper": 0.225,
-      "confidence": 0.95,
-      "bootstrap_samples": 10000
+  "metrics": [
+    {
+      "name": "subject_consistency",
+      "baseline": 0.8832832113063583,
+      "mugen": 0.8859599555454527,
+      "absolute_change": 0.002676744239094364,
+      "relative_error_reduction": 0.022933669346577767
+    },
+    {
+      "name": "motion_smoothness",
+      "baseline": 0.9820145276399405,
+      "mugen": 0.9824786442015501,
+      "absolute_change": 0.00046411656160960657
+    },
+    {
+      "name": "temporal_flickering",
+      "baseline": 0.9696340364802118,
+      "mugen": 0.9697767750305288,
+      "absolute_change": 0.0001427385503169898
     }
-  },
-  "completion_check": {
-    "checks": {
-      "held_out_pairs_30_to_50": true,
-      "all_pairs_have_b0_to_b3": true,
-      "uniform_generation_seed": true,
-      "required_metrics_present": true,
-      "side_by_side_cases_6_to_10": true
-    },
-    "passed": true
-  }
+  ]
 }

mugen_config.json

@@ -1,14 +1,14 @@
 {
-  "schema_version": 1,
+  "schema_version": 2,
   "base_model": "nvidia/AnyFlow-FAR-Wan2.1-1.3B-Diffusers",
-  "checkpoint_step": 300,
-  "condition_scale": 0.1,
+  "checkpoint_step": 10,
+  "condition_scale": 0.05,
+  "condition_token_count": 15,
+  "temporal_audio_tokens": 8,
+  "use_lora": false,
   "reference_tokens": 4,
   "reference_top_k": 3,
-  "lora": {
-    "rank": 16,
-    "alpha": 32
-  },
+  "lora": null,
   "generation": {
     "num_frames": 25,
     "height": 256,
@@ -17,12 +17,13 @@
   },
   "encoder_versions": {
     "imagebind": "ImageBind@53680b0",
-    "internvideo": "InternVideo@3965eef:Stage2-1B-224p-f4"
+    "internvideo": "InternVideo@3965eef:Stage2-1B-224p-f4",
+    "temporal_audio": "logmel-onset-v1:t8:m64:sr16000"
   },
   "feature_rows": 6000,
   "training_metrics": {
-    "train_loss": 0.23321709036827087,
-    "validation_loss": 0.4095851182937622
+    "train_loss": 0.43624502420425415,
+    "validation_loss": 0.3958585858345032
   },
   "upstream_weights_included": false,
   "optimizer_state_included": false