ZMF-FullBody-Lab / core.py
ZacharyFro18's picture
Upload 7 files
cf3ffb0 verified
Raw History Blame Contribute Delete
7.73 kB
"""ZMF full-body research pipeline. CPU stages do not download or run AI."""
from __future__ import annotations
import hashlib
import json
import math
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import uuid
import zipfile
MODEL_ID = 'Wan-AI/Wan2.2-Animate-14B-Diffusers'
MODEL_REVISION = '6f4df10861c758af86ac3c979aacc1bf5c03eff0'
FPS = 8
FRAMES = 17
WIDTH, HEIGHT = 384, 256
CONTROL_NAMES = ('src_pose.mp4', 'src_face.mp4', 'src_bg.mp4', 'src_mask.mp4')
def run(args, timeout=90):
p = subprocess.run([str(x) for x in args], capture_output=True, text=True, timeout=timeout)
if p.returncode:
raise ValueError('Media processing failed: ' + p.stderr[-1200:])
return p.stdout
def probe(path):
path = Path(path)
if not path.is_file() or path.stat().st_size > 300 * 1024 * 1024:
raise ValueError('Choose a local clip smaller than 300 MB.')
data = json.loads(run(['ffprobe', '-v', 'error', '-count_frames', '-select_streams', 'v:0',
'-show_entries', 'stream=width,height,avg_frame_rate,nb_read_frames,duration:format=duration',
'-of', 'json', path]))
if not data.get('streams'):
raise ValueError('No decodable video stream was found.')
s = data['streams'][0]
a, b = s.get('avg_frame_rate', '0/1').split('/')
fps = float(a) / max(float(b), 1)
duration = float(s.get('duration', data.get('format', {}).get('duration', 0)))
return dict(width=int(s['width']), height=int(s['height']), fps=fps,
frames=int(s.get('nb_read_frames', 0)), duration=duration)
def digest(path):
h = hashlib.sha256()
with open(path, 'rb') as f:
for block in iter(lambda: f.read(1024 * 1024), b''):
h.update(block)
return h.hexdigest()
def write_manifest(folder, values):
folder = Path(folder)
path = folder / 'job.json'
current = json.loads(path.read_text()) if path.exists() else {}
current.update(values)
tmp = folder / 'job.json.tmp'
tmp.write_text(json.dumps(current, indent=2))
os.replace(tmp, path)
return current
def prepare(clip, reference, root, start=0, permission=False):
if not permission:
raise ValueError('Confirm permission to use the footage and likeness.')
start = float(start)
if not math.isfinite(start) or start < 0:
raise ValueError('Start time must be a finite nonnegative number.')
info = probe(clip)
if info['duration'] < start + FRAMES / FPS:
raise ValueError('Select a range with at least 2.125 seconds remaining.')
# Preserve source aspect ratio with letterboxing; never stretch a camera-roll clip.
w, h = (HEIGHT, WIDTH) if info['height'] > info['width'] else (WIDTH, HEIGHT)
folder = Path(root) / str(uuid.uuid4())
folder.mkdir(parents=True)
write_manifest(folder, dict(schema='zmf-fullbody-dev-0.1', id=folder.name,
state='preparing', model=MODEL_ID, revision=MODEL_REVISION, trained_by_zmf=False,
fps=FPS, frames=FRAMES, width=w, height=h, start=start,
source_sha256=digest(clip), reference_sha256=digest(reference), permission=True,
payment_fallback=False, gpu_verified=False))
try:
scale = f'scale={w}:{h}:force_original_aspect_ratio=decrease,pad={w}:{h}:(ow-iw)/2:(oh-ih)/2,fps={FPS}'
run(['ffmpeg', '-y', '-v', 'error', '-ss', start, '-i', clip, '-vf', scale,
'-frames:v', FRAMES, '-an', '-c:v', 'libx264', '-pix_fmt', 'yuv420p', folder/'source.mp4'])
run(['ffmpeg', '-y', '-v', 'error', '-i', reference, '-vf',
f'scale={w}:{h}:force_original_aspect_ratio=decrease,pad={w}:{h}:(ow-iw)/2:(oh-ih)/2',
'-frames:v', '1', folder/'reference.png'])
run(['ffmpeg', '-y', '-v', 'error', '-i', folder/'source.mp4', '-frames:v', '1', folder/'first-frame.png'])
if probe(folder/'source.mp4')['frames'] != FRAMES:
raise ValueError('Decoder returned fewer frames than required.')
write_manifest(folder, {'state':'awaiting_controls'})
return folder
except Exception as e:
write_manifest(folder, {'state':'failed', 'error':str(e)})
raise
def validate_controls(folder):
folder = Path(folder)
j = json.loads((folder/'job.json').read_text())
for name in CONTROL_NAMES:
p = probe(folder/name)
if p['frames'] != j['frames'] or abs(p['fps'] - j['fps']) > .01:
raise ValueError(name + ' must match the prepared clip frame count and FPS.')
# Face crops intentionally have their own dimensions in the upstream format.
if name != 'src_face.mp4' and (p['width'], p['height']) != (j['width'], j['height']):
raise ValueError(name + ' has mismatched spatial dimensions.')
# A completely empty/full mask cannot meaningfully isolate a character.
raw = subprocess.run(['ffmpeg','-v','error','-i',str(folder/'src_mask.mp4'),
'-frames:v','1','-f','rawvideo','-pix_fmt','gray','-'], capture_output=True, check=True).stdout
fraction = sum(x > 127 for x in raw) / max(len(raw), 1)
if not .005 < fraction < .95:
raise ValueError('Mask must isolate the whole actor; white generates, black preserves.')
return j
def attach_controls(folder, files):
if len(files) != len(CONTROL_NAMES):
raise ValueError('Pose, face, background and mask controls are all required.')
folder = Path(folder)
write_manifest(folder, {'state':'validating_controls'})
try:
for src, name in zip(files, CONTROL_NAMES):
shutil.copyfile(src, folder/name)
validate_controls(folder)
write_manifest(folder, {'state':'ready_for_gpu'})
except Exception as e:
write_manifest(folder, {'state':'controls_invalid', 'error':str(e)})
raise
def checkpoint(folder):
folder = Path(folder)
output = folder.parent / (folder.name + '.zip')
with zipfile.ZipFile(output, 'w', zipfile.ZIP_DEFLATED) as z:
for p in folder.iterdir():
if p.is_file() and p.suffix in ('.mp4', '.png', '.json'):
z.write(p, p.name)
return output
def restore(archive, root):
folder = Path(root) / str(uuid.uuid4())
folder.mkdir(parents=True)
allowed = set(CONTROL_NAMES) | {'source.mp4','reference.png','first-frame.png','job.json','result.mp4'}
with zipfile.ZipFile(archive) as z:
if sum(i.file_size for i in z.infolist()) > 500 * 1024 * 1024:
raise ValueError('Checkpoint expands beyond 500 MB.')
if len(z.infolist()) > len(allowed):
raise ValueError('Too many checkpoint entries.')
for entry in z.infolist():
if entry.filename not in allowed:
raise ValueError('Unexpected checkpoint path.')
with z.open(entry) as src, open(folder/entry.filename, 'wb') as out:
shutil.copyfileobj(src, out)
j = json.loads((folder/'job.json').read_text())
if j.get('schema') != 'zmf-fullbody-dev-0.1' or j.get('model') != MODEL_ID or j.get('revision') != MODEL_REVISION:
raise ValueError('Unsupported checkpoint model/schema.')
if j.get('frames') != FRAMES or j.get('fps') != FPS or (j.get('width'),j.get('height')) not in [(WIDTH,HEIGHT),(HEIGHT,WIDTH)]:
raise ValueError('Checkpoint exceeds the fixed test limits.')
if not (folder/'source.mp4').is_file() or not (folder/'reference.png').is_file():
raise ValueError('Checkpoint is missing its source or reference.')
# Revalidate all controls before any future GPU request; never trust archived state.
write_manifest(folder, {'id':folder.name, 'state':'restored_unverified', 'gpu_verified':False})
return folder