File size: 6,302 Bytes
5cb23ae
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
"""Sample 16-frame I2V clips with Anima Turbo + the motion module through the ComfyUI API."""
import json, sys, time, urllib.request
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent))
from gen_h3 import API, DATA, post, upload  # noqa: E402

CASES = [("c0001", "var1", "seen"), ("c0006", "var2", "seen"), ("c0040", "var0", "new"), ("c0057", "var1", "new")]
MOTION = {"c0001": "waves his hand and laughs", "c0006": "raises one arm and stretches",
          "c0040": "turns her head toward the camera and smiles", "c0057": "waves her hand and laughs"}


def graph(image, prompt, motion, prefix, ref_image=None, mask_grow=None, denoise=1.0, mask_query="character", invert=False):
    model = ["1", 0]
    g = {
        "1": {"class_type": "UNETLoader", "inputs": {"unet_name": "anima/anima-turbo-v1.0-int8-convrot.safetensors", "weight_dtype": "default"}},
        "2": {"class_type": "CLIPLoader", "inputs": {"clip_name": "qwen_3_06b_base.safetensors", "type": "stable_diffusion", "device": "default"}},
        "3": {"class_type": "VAELoader", "inputs": {"vae_name": "qwen_image_vae.safetensors"}},
        "4": {"class_type": "CLIPTextEncode", "inputs": {"clip": ["2", 0], "text": prompt}},
        "5": {"class_type": "CLIPTextEncode", "inputs": {"clip": ["2", 0], "text": "worst quality, low quality, blurry, text, watermark"}},
        "6": {"class_type": "LoadImage", "inputs": {"image": image}},
        "7": {"class_type": "ImageScale", "inputs": {"image": ["6", 0], "upscale_method": "lanczos", "width": 384, "height": 576, "crop": "center"}},
        "8": {"class_type": "CosmosPredict2ImageToVideoLatent", "inputs": {"vae": ["3", 0], "width": 384, "height": 576, "length": 61, "batch_size": 1, "start_image": ["7", 0]}},
        "20": {"class_type": "KSampler", "inputs": {"positive": ["4", 0], "negative": ["5", 0], "latent_image": ["8", 0], "seed": 3, "steps": 8, "cfg": 1.0, "sampler_name": "euler", "scheduler": "simple", "denoise": 1.0}},
        "24": {"class_type": "LatentCutToBatch", "inputs": {"samples": ["20", 0], "dim": "t", "slice_size": 1}},
        "21": {"class_type": "VAEDecode", "inputs": {"vae": ["3", 0], "samples": ["24", 0]}},
        "22": {"class_type": "CreateVideo", "inputs": {"images": ["21", 0], "fps": 8.0}},
        "23": {"class_type": "SaveVideo", "inputs": {"video": ["22", 0], "filename_prefix": prefix, "format": "mp4", "codec": "h264"}},
    }
    if ref_image:  # R2V: character sheet through Anima Ref Adapter, no start image
        g["31"] = {"class_type": "LoadImage", "inputs": {"image": ref_image}}
        g["32"] = {"class_type": "AnimaRefAdapter", "inputs": {"model": model, "vae": ["3", 0], "image": ["31", 0], "lora_name": "anima_ref_adapter_v2_r64.safetensors",
                                                               "ref_guidance": 1.0, "correction": "none"}}
        model = ["32", 0]
        del g["8"]["inputs"]["start_image"]
    if mask_grow is not None:  # background frozen: keyframe latent in every frame, only the WeVisOne character matte is denoised
        g["40"] = {"class_type": "VAEEncode", "inputs": {"pixels": ["7", 0], "vae": ["3", 0]}}
        for i in range(41, 45):
            g[str(i)] = {"class_type": "LatentConcat", "inputs": {"samples1": [str(i - 1), 0], "samples2": [str(i - 1), 0], "dim": "t"}}
        g["45"] = {"class_type": "WeVisOneVLLoader", "inputs": {"clip_name": "wevisone_vl_4b_int8_convrot.safetensors"}}
        g["46"] = {"class_type": "WeVisOneVLPerceive", "inputs": {"clip": ["45", 0], "image": ["7", 0], "task": "matting", "query": mask_query,
                                                                  "max_length": 1024, "individual_masks": False}}
        g["47"] = {"class_type": "ThresholdMask", "inputs": {"mask": ["46", 3], "value": 0.3}}
        g["48"] = {"class_type": "GrowMask", "inputs": {"mask": ["47", 0], "expand": mask_grow, "tapered_corners": True}}
        mask = ["48", 0]
        if invert:  # redraw everything but the (grown) matte
            g["52"] = {"class_type": "InvertMask", "inputs": {"mask": mask}}
            mask = ["52", 0]
        g["49"] = {"class_type": "SetLatentNoiseMask", "inputs": {"samples": ["44", 0], "mask": mask}}
        g["50"] = {"class_type": "MaskToImage", "inputs": {"mask": mask}}
        g["51"] = {"class_type": "SaveImage", "inputs": {"images": ["50", 0], "filename_prefix": prefix + "_mask"}}
        g["20"]["inputs"]["latent_image"] = ["49", 0]
        g["20"]["inputs"]["denoise"] = denoise
        g["20"]["inputs"]["denoise"] = denoise
    if motion:
        g["30"] = {"class_type": "AnimaMotionModule", "inputs": {"model": model, "motion_name": motion}}
        model = ["30", 0]
    g["20"]["inputs"]["model"] = model
    return g


def main(motion, tag, r2v=False, mask_grow=None, denoise=1.0, mask_query="character", invert=False):
    for char, var, kind in CASES:
        meta = json.loads((DATA / char / "meta.json").read_text())
        prompt = f"masterpiece, best quality, score_7, safe, {meta['character_tags']}, {MOTION[char]}, anime screencap"
        if not meta["girl"]:
            prompt = prompt.replace("her ", "his ")
        image = upload(DATA / char / f"{var}.png")
        ref_image = upload(DATA / char / "ref.png") if r2v else None
        prefix = f"anima_motion_eval/{tag}_{kind}_{char}"
        pid = post("/prompt", json.dumps({"prompt": graph(image, prompt, motion, prefix, ref_image, mask_grow, denoise, mask_query, invert)}).encode(), {"Content-Type": "application/json"})["prompt_id"]
        t = time.time()
        while pid not in (h := json.load(urllib.request.urlopen(f"{API}/history/{pid}"))):
            time.sleep(2)
        s = h[pid]["status"]
        print(prefix, s["status_str"], f"{time.time() - t:.0f}s", flush=True)
        if s["status_str"] != "success":
            print(json.dumps(s["messages"])[-1200:])


if __name__ == "__main__":
    grow = next((int(a[5:]) for a in sys.argv[3:] if a.startswith("mask=")), None)
    denoise = next((float(a[8:]) for a in sys.argv[3:] if a.startswith("denoise=")), 1.0)
    query = next((a[6:] for a in sys.argv[3:] if a.startswith("query=")), "character")
    main(sys.argv[1] if sys.argv[1] != "none" else None, sys.argv[2], "r2v" in sys.argv[3:], grow, denoise, query, "invert" in sys.argv[3:])