viggle-animate: gpt-image-2 repaint + 4-step render

.gitattributes

@@ -68,3 +68,6 @@ examples/wushu_animals_out.mp4 filter=lfs diff=lfs merge=lfs -text
 examples/wushu_animals_ref.png filter=lfs diff=lfs merge=lfs -text
 examples/wushu_robots_out.mp4 filter=lfs diff=lfs merge=lfs -text
 examples/wushu_robots_ref.png filter=lfs diff=lfs merge=lfs -text
+examples/lipsync_drive.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/lipsync_out.mp4 filter=lfs diff=lfs merge=lfs -text
+examples/lipsync_ref.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -65,11 +65,16 @@ EX = Path(__file__).parent / "examples"
 # so clicking one costs no GPU and no OpenAI call. Several share a driving clip, hence the
 # name -> (drive file prefix, caption) shape: `corgi` and `clay` reuse `costume`'s clip, and
 # the second take of each two-character pair reuses its partner's.
-# Order is what a visitor reads top to bottom, so the swaps where the target is not a person go
-# first -- one paint and one render each, no best-of-N. The last three are the human cases the
-# model card compares against Wan2.2-Animate: the most useful, the least surprising to look at.
-# Within that, aspect ratio is grouped, because the output rows below are split on these indices.
+# Order is what a visitor reads top to bottom. `lipsync` is first because it is the only case
+# whose driving clip carries a soundtrack, and therefore the only one where the mouth has
+# anything to track: every other output's audio track is this finetune's digital silence
+# (-105 dBFS, against -22 for this one). Then the swaps where the target is not a person -- one
+# paint and one render each, no best-of-N -- and last the human cases the model card compares
+# against Wan2.2-Animate. Within that, aspect ratio is grouped, because the output rows below
+# are split on these indices.
 CASES = {
+    "lipsync": ("lipsync", "the driving clip has sound, so the mouth has something to say "
+                           "— turn the volume on"),
     "corgi": ("costume", "a corgi"),
     "panda": ("panda", "a giant panda"),
     "penguin": ("penguin", "a penguin"),
@@ -1015,12 +1020,15 @@ with gr.Blocks(title="Viggle-Animate") as demo:
         "Put the person from a still into a driving video. No pose estimation, no "
         "segmentation, no masks — the only preprocessing is repainting the clip's first "
         f"frame. 4 sampling steps. Takes {MIN_UPLOAD_S:.0f}–{MAX_FRAMES/FPS:.0f} s of driving "
-        f"video; anything longer is cut to {MAX_FRAMES/FPS:.1f} s.\n\n"
+        f"video; anything longer is cut to {MAX_FRAMES/FPS:.1f} s. If the clip carries a "
+        "soundtrack it is pinned into the render and the mouth tracks it; upload a silent clip "
+        "and the mouth moves without saying anything.\n\n"
         f"[Model card and weights]({MODEL_CARD}) · [viggle.ai]({SITE})"
     )
     with gr.Row():
         with gr.Column():
-            v = gr.Video(label="driving video — motion, framing and background are kept")
+            v = gr.Video(label="driving video — motion, framing and background are kept; "
+                                "its soundtrack, if it has one, is what the mouth follows")
             c = gr.Image(label="character — who to put in it", type="filepath")
             p = gr.Image(label="…or frame 0, already painted (skips gpt-image-2)",
                          type="filepath")
@@ -1051,12 +1059,13 @@ with gr.Blocks(title="Viggle-Animate") as demo:
     # actual evidence rather than a footnote.
     gr.Markdown("### What it produces\n"
                 "Every example above, rendered at 4 steps with seed 42. Each is a single paint "
-                "and a single render; only the last three swap in a person.")
+                "and a single render; the first and the last three swap in a person. Only the "
+                "first one has audio — it is the only one whose driving clip had any.")
     items = list(CASES.items())
-    # Rows are cut on aspect, not on a fixed width: six portrait swaps, then the four landscape
-    # two-character takes, then the three human cases. Mixing the two shapes in one row squeezes
-    # the portrait ones to a third of their height.
-    for row in (items[0:3], items[3:6], items[6:10], items[10:13]):
+    # Rows are cut on aspect, not on a fixed width: the landscape lip-sync take on its own, then
+    # six portrait swaps, the four landscape two-character takes, and the three human cases.
+    # Mixing the two shapes in one row squeezes the portrait ones to a third of their height.
+    for row in (items[0:1], items[1:4], items[4:7], items[7:11], items[11:14]):
         with gr.Row():
             for n, (_, caption) in row:
                 gr.Video(str(EX / f"{n}_out.mp4"), label=caption, interactive=False)

examples/lipsync_drive.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc3dd85a5be2e6246fb40afe46e337cb75d6c3558105fd81fa02f1ff286a5e92
+size 4223283

examples/lipsync_out.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0daf870fb10c109ecfbb33a313a16ec69e0e49bc4995fe5717557b6effeca15
+size 442574