LiquidInbetween 1.0: on-device photo in-betweener (Core ML FP16 + PyTorch weights)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+review.mp4 filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,28 @@
+MIT License
+
+Copyright (c) 2026 Haplo LLC
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+---
+
+The network architecture and starting weights are RIFE (ECCV 2022),
+https://github.com/hzwer/ECCV2022-RIFE, released under the MIT License:
+
+Copyright (c) 2021 hzwer

LiquidInbetweenNet.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e5b28063b63c4a923d7f0aba90a154a32e5876dd6e61018d3ab745e555b07f6
+size 191434

LiquidInbetweenNet.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a9e61af9ce4ffba98235445b31d9ba552bfa20ae217c281c0982112ead31720
+size 8782644

LiquidInbetweenNet.mlpackage/Manifest.json

@@ -0,0 +1,18 @@
+{
+    "fileFormatVersion": "1.0.0",
+    "itemInfoEntries": {
+        "637DC258-E1B3-4357-87ED-601746EF7735": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Weights",
+            "name": "weights",
+            "path": "com.apple.CoreML/weights"
+        },
+        "E7A6499F-D326-4DB5-87E9-7100DEA396D2": {
+            "author": "com.apple.CoreML",
+            "description": "CoreML Model Specification",
+            "name": "model.mlmodel",
+            "path": "com.apple.CoreML/model.mlmodel"
+        }
+    },
+    "rootModelIdentifier": "E7A6499F-D326-4DB5-87E9-7100DEA396D2"
+}

README.md

@@ -0,0 +1,123 @@
+---
+license: mit
+library_name: coreml
+tags:
+- frame-interpolation
+- video-frame-interpolation
+- coreml
+- ios
+- on-device
+- rife
+pipeline_tag: image-to-image
+---
+
+# LiquidInbetween
+
+A small learned in-betweener for photos that runs on the phone. Give it two
+photos and a moment between them, and it returns the frame at that moment.
+People are part-way into their next pose and things are part-way to where
+they went. Parts that appear or disappear are handled by the network, so
+nothing is smeared or double-exposed.
+
+![iOS 17+](https://img.shields.io/badge/iOS-17%2B-blue) ![Core ML](https://img.shields.io/badge/Core%20ML-8.6%20MB-green) ![MIT](https://img.shields.io/badge/license-MIT-lightgrey)
+
+## Use it in an app (Core ML)
+
+Add `LiquidInbetweenNet.mlpackage` to your Xcode target. The inputs are
+`first` and `second` (512 × 512 images) and `t` (a 1 × 1 × 1 × 1 float); the
+output `frame` is the 512 × 512 frame at that moment.
+
+```swift
+import CoreML
+import Vision
+
+let model = try LiquidInbetweenNet(configuration: MLModelConfiguration()).model
+let fill: [MLFeatureValue.ImageOption: Any] = [.cropAndScale: VNImageCropAndScaleOption.scaleFill.rawValue]
+let first = try MLFeatureValue(cgImage: photoA, pixelsWide: 512, pixelsHigh: 512,
+                               pixelFormatType: kCVPixelFormatType_32BGRA, options: fill)
+let second = try MLFeatureValue(cgImage: photoB, pixelsWide: 512, pixelsHigh: 512,
+                                pixelFormatType: kCVPixelFormatType_32BGRA, options: fill)
+let t = try MLMultiArray(shape: [1, 1, 1, 1], dataType: .float32)
+t[0] = 0.5
+let output = try model.prediction(from: MLDictionaryFeatureProvider(dictionary: [
+    "first": first, "second": second, "t": MLFeatureValue(multiArray: t),
+]))
+let frame = output.featureValue(for: "frame")?.imageBufferValue   // 512 × 512
+```
+
+Both photos are stretched to fill the square, so the motion between them
+stays consistent; stretch each frame back to your photos' shape to show it.
+
+## Use it in Python
+
+```python
+import torch
+from safetensors.torch import load_file
+from liquid_model import IFNet
+
+net = IFNet(timestep=True).eval()
+net.load_state_dict(load_file("liquid_inbetween.safetensors"))
+# first, second: 1 × 3 × H × W tensors in 0…1, sides divisible by 32.
+with torch.no_grad():
+    frame = net(first, second, t=torch.tensor([[[[0.5]]]]))
+```
+
+## The model
+
+- **Architecture.** The model is RIFE's IFNet (Huang et al., *Real-Time
+  Intermediate Flow Estimation for Video Frame Interpolation*, ECCV 2022,
+  MIT). It predicts flows from the in-between moment back to both photos,
+  plus an occlusion mask that decides which photo each pixel trusts. It has
+  about 2.3 M parameters.
+- **What's new: any moment in one pass.** Every block also sees the timestep
+  t, and the mirrored pass sees 1 − t. RIFE alone only makes the exact
+  middle and needs recursion for other moments. The new input channel starts
+  at zero, so training begins exactly at RIFE's behaviour.
+- **Fine-tuned for photos, not video.** Each training pair is two frames
+  0.13–0.8 s apart (photos taken moments apart), much larger gaps than
+  consecutive video frames. The target is a real frame somewhere between
+  them, at its own t. The footage is public-domain and CC0 Wikimedia
+  Commons video of people moving: dance, children playing, NASA astronaut
+  reels and military PT. See `training/SOURCES.tsv`.
+- **Core ML.** The weights are FP16, about 8.6 MB, with image inputs and
+  output. The conversion was measured at 42.9 dB PSNR against the PyTorch
+  model. 8-bit weights halved the size but cost too much accuracy, so they
+  are not used.
+
+Training, evaluation and export scripts are in `training/`.
+
+## Files
+
+- `LiquidInbetweenNet.mlpackage`: Core ML, FP16. Inputs are `first` and
+  `second` (512 × 512 RGB images) and `t` (1×1×1×1 float); the output is
+  `frame`.
+- `liquid_inbetween.safetensors`: PyTorch weights for `IFNet(timestep=True)`
+  in `liquid_model.py`.
+- `review.mp4`: the model's frames beside a plain cross-fade, on held-out
+  footage.
+
+## Results on held-out footage
+
+The evaluation set is a different video from any used in training: a
+public-domain folk dance, with gaps of 0.4–0.8 s. Scores are PSNR (dB) at
+t = ¼, ½ and ¾.
+
+| | Whole frame | Moving pixels only |
+|---|---|---|
+| Plain cross-fade | 36.97 | 22.74 |
+| RIFE (recursive for ¼, ¾) | 36.50 | 22.60 |
+| **LiquidInbetween** (one pass) | **36.77** | **22.92** |
+
+PSNR rewards blurry cross-fades, so it undersells sharp in-betweens: compare
+`review.mp4`.
+
+## Limits
+
+This is an in-betweener, not a generative video model. It moves what is in
+the two photos and does not invent content that is in neither. Two
+unrelated photos become a flowing dissolve, not a semantic morph.
+
+## License
+
+MIT, as is RIFE's code and weights it starts from (© 2021 hzwer). Made by
+[Haplo LLC](https://haploapp.com).

liquid_inbetween.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24544362f618979594c82aa641e599f94ac9fc10d556f81c1c7aea4451670a14
+size 9126276

liquid_model.py

@@ -0,0 +1,119 @@
+"""LiquidInbetween: a small learned in-betweener for photos.
+
+The architecture is RIFE's IFNet (Huang et al., ECCV 2022, MIT licence,
+github.com/hzwer/ECCV2022-RIFE), extended with a timestep input so one pass
+gives the frame at any t in (0, 1), not only the middle. The extra channel
+starts at zero, so training begins exactly at RIFE's behaviour at t = 0.5.
+"""
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+def conv(cin, cout, k=3, s=1, p=1):
+    return nn.Sequential(nn.Conv2d(cin, cout, k, s, p, bias=True), nn.PReLU(cout))
+
+
+def base_grid(n, h, w, device, dtype):
+    gx = torch.linspace(-1.0, 1.0, w, device=device, dtype=dtype).view(1, 1, 1, w).expand(n, -1, h, -1)
+    gy = torch.linspace(-1.0, 1.0, h, device=device, dtype=dtype).view(1, 1, h, 1).expand(n, -1, -1, w)
+    return torch.cat([gx, gy], 1)
+
+
+def warp(img, flow):
+    """Backward warp: each output pixel reads `img` at its position + `flow` (pixels)."""
+    n, _, h, w = flow.shape
+    f = torch.cat([flow[:, 0:1] / ((w - 1.0) / 2.0), flow[:, 1:2] / ((h - 1.0) / 2.0)], 1)
+    # Clamping the grid gives border padding on every backend (MPS and Core ML too).
+    g = (base_grid(n, h, w, flow.device, flow.dtype) + f).clamp(-1.0, 1.0).permute(0, 2, 3, 1)
+    return F.grid_sample(img, g, mode="bilinear", padding_mode="zeros", align_corners=True)
+
+
+class IFBlock(nn.Module):
+    def __init__(self, in_planes, c=90):
+        super().__init__()
+        self.conv0 = nn.Sequential(conv(in_planes, c // 2, 3, 2, 1), conv(c // 2, c, 3, 2, 1))
+        self.convblock0 = nn.Sequential(conv(c, c), conv(c, c))
+        self.convblock1 = nn.Sequential(conv(c, c), conv(c, c))
+        self.convblock2 = nn.Sequential(conv(c, c), conv(c, c))
+        self.convblock3 = nn.Sequential(conv(c, c), conv(c, c))
+        self.conv1 = nn.Sequential(nn.ConvTranspose2d(c, c // 2, 4, 2, 1), nn.PReLU(c // 2), nn.ConvTranspose2d(c // 2, 4, 4, 2, 1))
+        self.conv2 = nn.Sequential(nn.ConvTranspose2d(c, c // 2, 4, 2, 1), nn.PReLU(c // 2), nn.ConvTranspose2d(c // 2, 1, 4, 2, 1))
+
+    def forward(self, x, flow, scale):
+        x = F.interpolate(x, scale_factor=1.0 / scale, mode="bilinear", align_corners=False)
+        flow = F.interpolate(flow, scale_factor=1.0 / scale, mode="bilinear", align_corners=False) / scale
+        feat = self.conv0(torch.cat((x, flow), 1))
+        feat = self.convblock0(feat) + feat
+        feat = self.convblock1(feat) + feat
+        feat = self.convblock2(feat) + feat
+        feat = self.convblock3(feat) + feat
+        flow = F.interpolate(self.conv1(feat), scale_factor=scale, mode="bilinear", align_corners=False) * scale
+        mask = F.interpolate(self.conv2(feat), scale_factor=scale, mode="bilinear", align_corners=False)
+        return flow, mask
+
+
+class IFNet(nn.Module):
+    """RIFE's IFNet. With `timestep`, every block also sees t (and the
+    mirrored pass 1 − t), so the frame at any t comes out in one pass."""
+
+    def __init__(self, timestep=False, c=90):
+        super().__init__()
+        self.timestep = timestep
+        extra = 1 if timestep else 0
+        self.block0 = IFBlock(7 + 4 + extra, c)
+        self.block1 = IFBlock(7 + 4 + extra, c)
+        self.block2 = IFBlock(7 + 4 + extra, c)
+
+    def forward(self, img0, img1, t=None, all_scales=False):
+        n, _, h, w = img0.shape
+        flow = torch.zeros(n, 4, h, w, device=img0.device, dtype=img0.dtype)
+        mask = torch.zeros(n, 1, h, w, device=img0.device, dtype=img0.dtype)
+        if self.timestep:
+            tt = (t if t is not None else torch.full((n, 1, 1, 1), 0.5, device=img0.device, dtype=img0.dtype)).expand(n, 1, h, w)
+        w0, w1 = img0, img1
+        outputs = []
+        for block, scale in ((self.block0, 4), (self.block1, 2), (self.block2, 1)):
+            a = [w0, w1, mask] + ([tt] if self.timestep else [])
+            b = [w1, w0, -mask] + ([1 - tt] if self.timestep else [])
+            f0, m0 = block(torch.cat(a, 1), flow, scale)
+            f1, m1 = block(torch.cat(b, 1), torch.cat((flow[:, 2:4], flow[:, :2]), 1), scale)
+            flow = flow + (f0 + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2
+            mask = mask + (m0 - m1) / 2
+            w0 = warp(img0, flow[:, :2])
+            w1 = warp(img1, flow[:, 2:4])
+            if all_scales:
+                m = torch.sigmoid(mask)
+                outputs.append(w0 * m + w1 * (1 - m))
+        m = torch.sigmoid(mask)
+        out = w0 * m + w1 * (1 - m)
+        return outputs if all_scales else out
+
+
+def load_teacher(path):
+    """RIFE's ECCV 2022 weights (safetensors) into IFNet()."""
+    from safetensors.torch import load_file
+    net = IFNet(timestep=False)
+    state = load_file(path)
+    state = {k.replace("module.", ""): v for k, v in state.items() if not k.startswith("block_tea")}
+    missing, unexpected = net.load_state_dict(state, strict=False)
+    assert not missing, missing
+    return net
+
+
+def student_from_teacher(teacher):
+    """IFNet(timestep=True) starting exactly as the teacher: the new t channel's
+    weights are zero."""
+    student = IFNet(timestep=True)
+    sd = student.state_dict()
+    for k, v in teacher.state_dict().items():
+        if k.endswith("conv0.0.0.weight"):
+            new = torch.zeros_like(sd[k])
+            # Inputs: warped0 (3), warped1 (3), mask (1) | t (1) | flow (4)
+            new[:, :7] = v[:, :7]
+            new[:, 8:] = v[:, 7:]
+            sd[k] = new
+        else:
+            sd[k] = v
+    student.load_state_dict(sd)
+    return student

review.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4fa2ba32374cf6df884bba73b830cfa6de043d628ff68535b4ff8976f7fc766
+size 7395081

training/SOURCES.tsv

@@ -0,0 +1,10 @@
+v00.webm	File:The Bakweri Mola dance.webm	CC0 https://commons.wikimedia.org/wiki/File:The_Bakweri_Mola_dance.webm
+v01.webm	File:NASA Astronaut Candidate Ben Bailey Resource Reel (jsc2025m000154).webm	Public domain https://commons.wikimedia.org/wiki/File:NASA_Astronaut_Candidate_Ben_Bailey_Resource_Reel_(jsc2025m000154).webm
+v02.webm	File:NASA Astronaut Candidate Anna Menon Resource Reel (jsc2025m000153).webm	Public domain https://commons.wikimedia.org/wiki/File:NASA_Astronaut_Candidate_Anna_Menon_Resource_Reel_(jsc2025m000153).webm
+v03.webm	File:NASA Astronaut Candidate Imelda Muller Resource Reel (jsc2025m000157).webm	Public domain https://commons.wikimedia.org/wiki/File:NASA_Astronaut_Candidate_Imelda_Muller_Resource_Reel_(jsc2025m000157).webm
+v04.webm	File:Dali-Girls-playing-20241229.webm	CC0 https://commons.wikimedia.org/wiki/File:Dali-Girls-playing-20241229.webm
+v05.webm	File:B-Roll- SETAF-AF leads physical training during sergeants time (994732).webm	Public_domain https://commons.wikimedia.org/wiki/File:B-Roll-_SETAF-AF_leads_physical_training_during_sergeants_time_(994732).webm
+v06.webm	File:Valiant Mark 25- MRF-SEA, SAF conduct physical training (956293).webm	Public_domain https://commons.wikimedia.org/wiki/File:Valiant_Mark_25-_MRF-SEA,_SAF_conduct_physical_training_(956293).webm
+
+# Held out for evaluation (never trained on)
+v00 (eval)	File:Folk dance of Austria Tyrol.webm	Public domain https://commons.wikimedia.org/wiki/File:Folk_dance_of_Austria_Tyrol.webm

training/check_coreml.py

@@ -0,0 +1,25 @@
+"""Compare a Core ML export with the PyTorch model on a real pair."""
+import sys
+from pathlib import Path
+import coremltools as ct
+import numpy as np
+import torch
+from PIL import Image
+sys.path.insert(0, str(Path(__file__).parent))
+from liquid_model import IFNet, load_teacher, student_from_teacher
+
+weights, package, a_path, b_path, out_png = sys.argv[1:6]
+net = student_from_teacher(load_teacher(Path(weights))) if weights.endswith(".safetensors") else IFNet(timestep=True)
+if not weights.endswith(".safetensors"):
+    net.load_state_dict(torch.load(weights, map_location="cpu"))
+net.eval()
+a = Image.open(a_path).convert("RGB").resize((512, 512), Image.LANCZOS)
+b = Image.open(b_path).convert("RGB").resize((512, 512), Image.LANCZOS)
+model = ct.models.MLModel(package)
+coreml = np.asarray(model.predict({"first": a, "second": b, "t": np.full((1, 1, 1, 1), 0.5, np.float32)})["frame"].convert("RGB")).astype(np.float32)
+ta, tb = (torch.from_numpy(np.asarray(x)).permute(2, 0, 1)[None].float() / 255 for x in (a, b))
+with torch.no_grad():
+    ref = (net(ta, tb, torch.full((1, 1, 1, 1), 0.5)).clamp(0, 1)[0].permute(1, 2, 0).numpy() * 255)
+mse = ((coreml - ref) ** 2).mean()
+print(f"Core ML vs PyTorch PSNR: {10 * np.log10(255 ** 2 / max(mse, 1e-9)):.2f} dB")
+Image.fromarray(np.concatenate([ref, coreml], 1).astype(np.uint8)).save(out_png)

training/eval_motion.py

@@ -0,0 +1,59 @@
+"""Score in-betweens where things move: PSNR over pixels whose two endpoint
+frames differ (dilated), on held-out triplets, for a blend, RIFE and ours."""
+import json, random, sys
+from pathlib import Path
+import numpy as np
+import torch
+import torch.nn.functional as F
+sys.path.insert(0, str(Path(__file__).parent))
+from liquid_model import IFNet, load_teacher
+
+eval_dir, teacher_path, ours_path = Path(sys.argv[1]), Path(sys.argv[2]), Path(sys.argv[3])
+device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")
+index = json.loads((eval_dir / "index.json").read_text())
+videos = [(np.load(eval_dir / f"{v['name']}.npy", mmap_mode="r"), set(v["cuts"])) for v in index]
+teacher = load_teacher(teacher_path).to(device).eval()
+ours = IFNet(timestep=True).to(device).eval()
+ours.load_state_dict(torch.load(ours_path, map_location=device))
+
+random.seed(99)
+samples = []
+while len(samples) < 90:
+    frames, cuts = random.choice(videos)
+    gap = random.choice([6, 8, 10, 12])
+    i = random.randrange(0, len(frames) - gap)
+    if any(i < c <= i + gap for c in cuts):
+        continue
+    j = random.choice([gap // 4, gap // 2, 3 * gap // 4])
+    samples.append((frames[i], frames[i + j], frames[i + gap], j / gap))
+
+def tensor(f):
+    return torch.from_numpy(np.ascontiguousarray(f)).permute(2, 0, 1)[None].float().div(255).to(device)
+
+def rife_at(t, x0, x1, lo=0.0, hi=1.0, depth=0):
+    mid = teacher(x0, x1)
+    c = (lo + hi) / 2
+    if abs(c - t) < 1e-6 or depth > 4:
+        return mid
+    return rife_at(t, x0, mid, lo, c, depth + 1) if t < c else rife_at(t, mid, x1, c, hi, depth + 1)
+
+scores = {"blend": [], "rife": [], "ours": []}
+with torch.no_grad():
+    for a, m, b, t in samples:
+        A, M, B = tensor(a), tensor(m), tensor(b)
+        h, w = A.shape[-2:]
+        H, W = (h + 31) // 32 * 32, (w + 31) // 32 * 32
+        pad = lambda x: F.pad(x, (0, W - w, 0, H - h), mode="replicate")
+        moving = ((A - B).abs().max(1, keepdim=True)[0] > 0.08).float()
+        moving = (F.max_pool2d(moving, 9, 1, 4) > 0).float()
+        if moving.mean() < 0.01:
+            continue
+        outs = {
+            "blend": A * (1 - t) + B * t,
+            "rife": rife_at(t, pad(A), pad(B))[..., :h, :w],
+            "ours": ours(pad(A), pad(B), torch.tensor([[[[t]]]], device=device))[..., :h, :w],
+        }
+        for k, o in outs.items():
+            mse = (((o.clamp(0, 1) - M) ** 2) * moving).sum() / (moving.sum() * 3)
+            scores[k].append(-10 * torch.log10(mse + 1e-10).item())
+print({k: round(float(np.mean(v)), 2) for k, v in scores.items()}, "on", len(scores["ours"]), "moving samples")

training/export_coreml.py

@@ -0,0 +1,77 @@
+"""Export LiquidInbetween to Core ML.
+
+Images in and out (RGB, 512 × 512): pass two pixel buffers and a moment
+t ∈ (0, 1), get the frame at that moment. Weights are FP16 (about 8.6 MB).
+
+    python export_coreml.py <weights.pt | teacher.safetensors> <out.mlpackage> [--int8]
+"""
+import sys
+from pathlib import Path
+
+import coremltools as ct
+import coremltools.optimize.coreml as cto
+import numpy as np
+import torch
+
+sys.path.insert(0, str(Path(__file__).parent))
+from liquid_model import IFNet, load_teacher, student_from_teacher
+
+SIZE = 512
+
+
+class Wrapper(torch.nn.Module):
+    """Pixel values 0...255 in and out, as Core ML image types expect."""
+
+    def __init__(self, net):
+        super().__init__()
+        self.net = net
+
+    def forward(self, first, second, t):
+        out = self.net(first / 255.0, second / 255.0, t)
+        return (out.clamp(0, 1) * 255.0)
+
+
+weights, out = Path(sys.argv[1]), Path(sys.argv[2])
+if weights.suffix == ".safetensors":
+    net = student_from_teacher(load_teacher(weights))
+else:
+    net = IFNet(timestep=True)
+    net.load_state_dict(torch.load(weights, map_location="cpu"))
+model = Wrapper(net.eval()).eval()
+
+example = (torch.rand(1, 3, SIZE, SIZE) * 255, torch.rand(1, 3, SIZE, SIZE) * 255, torch.full((1, 1, 1, 1), 0.5))
+with torch.no_grad():
+    traced = torch.jit.trace(model, example)
+
+mlmodel = ct.convert(
+    traced,
+    inputs=[
+        ct.ImageType(name="first", shape=(1, 3, SIZE, SIZE), color_layout=ct.colorlayout.RGB),
+        ct.ImageType(name="second", shape=(1, 3, SIZE, SIZE), color_layout=ct.colorlayout.RGB),
+        ct.TensorType(name="t", shape=(1, 1, 1, 1), dtype=np.float32),
+    ],
+    outputs=[ct.ImageType(name="frame", color_layout=ct.colorlayout.RGB)],
+    convert_to="mlprogram",
+    compute_precision=ct.precision.FLOAT16,
+    minimum_deployment_target=ct.target.iOS17,
+)
+if "--int8" in sys.argv:
+    # Experimental: 8-bit weights halve the size, but measured 24.7 dB against
+    # PyTorch (FP16 alone: 42.9 dB), visibly worse, so FP16 is the default.
+    # The transposed convolutions are the flow and mask heads: a tiny weight
+    # error there becomes pixels of motion error, so they stay FP16.
+    config = cto.OptimizationConfig(
+        global_config=cto.OpLinearQuantizerConfig(mode="linear_symmetric", dtype="int8", granularity="per_channel"),
+        op_type_configs={"conv_transpose": None},
+    )
+    mlmodel = cto.linear_quantize_weights(mlmodel, config)
+mlmodel.short_description = "LiquidInbetween: the frame at moment t between two photos."
+mlmodel.author = "Haplo LLC (jc_builds). Based on RIFE (Huang et al., ECCV 2022, MIT)."
+mlmodel.license = "MIT"
+mlmodel.version = "1.0"
+mlmodel.input_description["first"] = "The first photo, 512 × 512 RGB."
+mlmodel.input_description["second"] = "The second photo, 512 × 512 RGB."
+mlmodel.input_description["t"] = "The moment between them, 0...1."
+mlmodel.output_description["frame"] = "The frame at moment t."
+mlmodel.save(str(out))
+print("saved", out)

training/prepare_frames.py

@@ -0,0 +1,25 @@
+"""Decode training videos to 15 fps frames (fit in 448 px) as uint8 arrays,
+and mark scene cuts so no training triplet spans one."""
+import json, subprocess, sys
+from pathlib import Path
+import numpy as np
+
+src, out = Path(sys.argv[1]), Path(sys.argv[2])
+out.mkdir(parents=True, exist_ok=True)
+index = []
+for video in sorted(src.glob("v*.webm")):
+    probe = json.loads(subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
+                                       "stream=width,height", "-of", "json", str(video)], capture_output=True, text=True).stdout)
+    w, h = probe["streams"][0]["width"], probe["streams"][0]["height"]
+    s = 448 / max(w, h)
+    W, H = int(round(w * s / 8) * 8), int(round(h * s / 8) * 8)
+    raw = subprocess.run(["ffmpeg", "-v", "error", "-i", str(video), "-vf", f"fps=15,scale={W}:{H}:flags=area",
+                          "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], capture_output=True).stdout
+    frames = np.frombuffer(raw, np.uint8).reshape(-1, H, W, 3)
+    small = frames[:, ::8, ::8].astype(np.float32).mean(-1)
+    diff = np.abs(np.diff(small, axis=0)).mean((1, 2))
+    cuts = np.where(diff > max(18.0, np.median(diff) * 6))[0] + 1   # first frame of each new shot
+    np.save(out / f"{video.stem}.npy", frames)
+    index.append({"name": video.stem, "frames": int(len(frames)), "width": W, "height": H, "cuts": cuts.tolist()})
+    print(video.stem, frames.shape, "cuts:", len(cuts))
+(out / "index.json").write_text(json.dumps(index, indent=1))

training/train.py

@@ -0,0 +1,165 @@
+"""Fine-tune LiquidInbetween from RIFE's weights.
+
+Each sample is two frames 2–12 steps apart at 15 fps (0.13–0.8 s: photos
+taken moments apart), and a real frame somewhere between them as the target
+at its own t. The held-out set is a different video entirely.
+
+    python train.py <frames dir> <eval frames dir> <teacher.safetensors> <out dir> [iterations]
+"""
+import json, math, random, sys, time
+from pathlib import Path
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+
+sys.path.insert(0, str(Path(__file__).parent))
+from liquid_model import load_teacher, student_from_teacher
+
+frames_dir, eval_dir, teacher_path, out_dir = map(Path, sys.argv[1:5])
+iterations = int(sys.argv[5]) if len(sys.argv) > 5 else 6000
+out_dir.mkdir(parents=True, exist_ok=True)
+device = torch.device("mps")
+random.seed(7)
+np.random.seed(7)
+torch.manual_seed(7)
+
+
+def load_set(d):
+    index = json.loads((d / "index.json").read_text())
+    return [(np.load(d / f"{v['name']}.npy", mmap_mode="r"), set(v["cuts"])) for v in index]
+
+
+train_set, eval_set = load_set(frames_dir), load_set(eval_dir)
+
+
+def triplet(videos, gap_range, size=None):
+    """Two frames `gap` apart with no cut between them, and one in between."""
+    while True:
+        frames, cuts = random.choices(videos, weights=[len(v[0]) for v in videos])[0]
+        gap = random.randint(*gap_range)
+        if len(frames) <= gap:
+            continue
+        i = random.randrange(0, len(frames) - gap)
+        if any(i < c <= i + gap for c in cuts):
+            continue
+        j = random.randint(1, gap - 1)
+        a, m, b = frames[i], frames[i + j], frames[i + gap]
+        t = j / gap
+        if size:
+            h, w = a.shape[:2]
+            y, x = random.randrange(0, h - size + 1), random.randrange(0, w - size + 1)
+            a, m, b = (f[y:y + size, x:x + size] for f in (a, m, b))
+        return a, m, b, t
+
+
+def to_tensor(arrays):
+    return torch.from_numpy(np.stack(arrays)).permute(0, 3, 1, 2).float().div(255)
+
+
+def batch(n, size=224):
+    items = []
+    for _ in range(n):
+        a, m, b, t = triplet(train_set, (2, 12), size)
+        if random.random() < 0.5:
+            a, m, b = a[:, ::-1], m[:, ::-1], b[:, ::-1]
+        if random.random() < 0.5:
+            a, b, t = b, a, 1 - t
+        items.append((np.ascontiguousarray(a), np.ascontiguousarray(m), np.ascontiguousarray(b), t))
+    a, m, b = (to_tensor([x[k] for x in items]).to(device) for k in range(3))
+    t = torch.tensor([x[3] for x in items], dtype=torch.float32, device=device).view(-1, 1, 1, 1)
+    return a, m, b, t
+
+
+def charbonnier(x, y):
+    return torch.sqrt((x - y) ** 2 + 1e-6).mean()
+
+
+def laplacian_loss(x, y, levels=4):
+    loss = 0
+    for _ in range(levels):
+        xd, yd = F.avg_pool2d(x, 2), F.avg_pool2d(y, 2)
+        loss = loss + (x - F.interpolate(xd, size=x.shape[-2:], mode="bilinear", align_corners=False)
+                       - (y - F.interpolate(yd, size=y.shape[-2:], mode="bilinear", align_corners=False))).abs().mean()
+        x, y = xd, yd
+    return loss
+
+
+teacher = load_teacher(teacher_path).to(device).eval()
+student = student_from_teacher(load_teacher(teacher_path)).to(device)
+for p in teacher.parameters():
+    p.requires_grad_(False)
+
+# Fixed held-out triplets: a different video, larger gaps, three moments.
+random.seed(123)
+held = []
+while len(held) < 120:
+    a, m, b, t = triplet(eval_set, (6, 10))
+    if t in (0.25, 0.5, 0.75) or len(held) % 3 == 0:
+        held.append((a, m, b, t))
+random.seed(7)
+
+
+def psnr(x, y):
+    return (-10 * torch.log10(((x - y) ** 2).mean(dim=(1, 2, 3)) + 1e-10)).mean().item()
+
+
+@torch.no_grad()
+def evaluate(net, timestep):
+    scores = {"blend": [], "model": []}
+    for a, m, b, t in held:
+        a_, m_, b_ = (to_tensor([np.ascontiguousarray(f)]).to(device) for f in (a, m, b))
+        h, w = a_.shape[-2:]
+        H, W = (h + 31) // 32 * 32, (w + 31) // 32 * 32
+        pad = lambda x: F.pad(x, (0, W - w, 0, H - h), mode="replicate")
+        if timestep:
+            out = net(pad(a_), pad(b_), torch.tensor([[[[t]]]], device=device))
+        else:
+            # RIFE only makes the middle; reach other moments by halving recursively.
+            def at(lo, hi, x0, x1, depth=0):
+                mid = net(x0, x1)
+                c = (lo + hi) / 2
+                if abs(c - t) < 1e-6 or depth > 4:
+                    return mid
+                return at(lo, c, x0, mid, depth + 1) if t < c else at(c, hi, mid, x1, depth + 1)
+            out = at(0.0, 1.0, pad(a_), pad(b_))
+        out = out[..., :h, :w].clamp(0, 1)
+        scores["model"].append(psnr(out, m_))
+        scores["blend"].append(psnr(a_ * (1 - t) + b_ * t, m_))
+    return {k: float(np.mean(v)) for k, v in scores.items()}
+
+
+log = []
+baseline = evaluate(teacher, timestep=False)
+start = evaluate(student, timestep=True)
+print(f"held-out PSNR: blend {baseline['blend']:.2f}, RIFE {baseline['model']:.2f}, ours at start {start['model']:.2f}", flush=True)
+log.append({"iteration": 0, "blend": baseline["blend"], "rife": baseline["model"], "ours": start["model"]})
+
+optimizer = torch.optim.AdamW(student.parameters(), lr=1e-4, weight_decay=1e-4)
+best = start["model"]
+began = time.time()
+for it in range(1, iterations + 1):
+    lr = 1e-5 + 0.5 * (1e-4 - 1e-5) * (1 + math.cos(math.pi * it / iterations))
+    for g in optimizer.param_groups:
+        g["lr"] = lr
+    a, m, b, t = batch(8)
+    outs = student(a, b, t, all_scales=True)
+    loss = charbonnier(outs[-1], m) + 0.5 * laplacian_loss(outs[-1], m) + 0.25 * charbonnier(outs[-2], m)
+    optimizer.zero_grad(set_to_none=True)
+    loss.backward()
+    torch.nn.utils.clip_grad_norm_(student.parameters(), 1.0)
+    optimizer.step()
+    if it % 100 == 0:
+        print(f"it {it} loss {loss.item():.4f} lr {lr:.2e} {(time.time() - began) / it:.2f}s/it", flush=True)
+    if it % 500 == 0 or it == iterations:
+        student.eval()
+        score = evaluate(student, timestep=True)["model"]
+        student.train()
+        log.append({"iteration": it, "ours": score})
+        print(f"== it {it}: held-out PSNR ours {score:.2f} (RIFE {baseline['model']:.2f}, blend {baseline['blend']:.2f})", flush=True)
+        if score > best:
+            best = score
+            torch.save(student.state_dict(), out_dir / "liquid_inbetween.pt")
+        (out_dir / "log.json").write_text(json.dumps(log, indent=1))
+torch.save(student.state_dict(), out_dir / "liquid_inbetween_last.pt")
+print("done; best held-out PSNR", best, flush=True)