# -*- coding: utf-8 -*- """H3 multishot utilities - JoyEcho-style single-script prompting for the MiniMax H3 chained workflow. One node, no dependencies. Accepts the same script formats the JoyEcho stack uses: - JSON: {"prompts": ["shot 1 ...", "shot 2 ...", "shot 3 ..."]} - plain text with --- separators between shots Feeds up to 4 shot prompts as separate STRING outputs. Missing shots fall back to the previous shot's prompt so a 2-shot script still runs a 3-shot graph without erroring. """ import json import re def _repair_json(text): """Parse JSON, auto-closing unterminated brackets/quotes. Long multi-prompt scripts get truncated or lose their final brace all the time (a 4,500-char script with the closing '}' missing is not a typo the author can see). Returns (data, note): data is None on real failure and note carries the error; note is a description when a repair was applied, or "" when the text parsed clean. """ try: return json.loads(text), "" except json.JSONDecodeError as e: first_err = str(e) # bind now; Python clears the except-name on exit # walk the text tracking string state, then close what is still open stack, in_str, esc = [], False, False for ch in text: if in_str: if esc: esc = False elif ch == "\\": esc = True elif ch == '"': in_str = False continue if ch == '"': in_str = True elif ch in "{[": stack.append(ch) elif ch in "}]": if stack and ((ch == "}" and stack[-1] == "{") or (ch == "]" and stack[-1] == "[")): stack.pop() candidate = text.rstrip() fixes = [] if in_str: candidate += '"' fixes.append("closed an open string") if candidate.endswith(","): candidate = candidate[:-1] fixes.append("dropped a trailing comma") # trailing comma before a closer, e.g. ["a","b",] or {"k":1,} cleaned = re.sub(r",(\s*[\]}])", r"\1", candidate) if cleaned != candidate: candidate = cleaned fixes.append("removed comma(s) before a closing bracket") for opener in reversed(stack): candidate += "}" if opener == "{" else "]" if stack: fixes.append("added " + "".join("}" if o == "{" else "]" for o in reversed(stack))) if not fixes: return None, first_err try: return json.loads(candidate), ", ".join(fixes) except json.JSONDecodeError as e: return None, str(e) _AT_NBANDS = 8 def _up_model_list(): try: import folder_paths return folder_paths.get_filename_list("upscale_models") except Exception: return [] def _load_up_model(name): """Load an upscale model by NAME, through ComfyUI's own loader node. Do not reimplement this. The first version here copied the loader's body from an older ComfyUI - state dict, spandrel, eval() - and missed that the current one also attaches a CoreModelPatcher, which ImageUpscaleWithModel then reads as upscale_model.patcher.load_device. It crashed AFTER shot 1 had rendered. Calling the real node means this cannot drift out of sync with whatever ComfyUI does next. """ from comfy_extras.nodes_upscale_model import UpscaleModelLoader out = UpscaleModelLoader().load_model(name) # the V3 node API returns a NodeOutput; older returns a plain tuple for attr in ("result", "results"): if hasattr(out, attr): out = getattr(out, attr) break while isinstance(out, (tuple, list)): out = out[0] if not hasattr(out, "patcher"): raise RuntimeError( "upscale_model_name=%r loaded, but the result has no .patcher - " "ComfyUI's upscale loader has changed shape again. Wire a Load " "Upscale Model node instead, or set upscale_model_name to " "(none)." % name) return out def _mn_normalize(parts, mode, med=9): """Level luma across the WHOLE assembled chain, to ONE global target. Deflicker over the finished timeline. Two design points, both learned the hard way: * ONE GLOBAL target, not a rolling or smoothed-local one. The per-shot colour mode drives each shot to a rolling house and leaves a hard step at every join (measured 29% warmth). A smoothed-local target is no better for chained video: a shot boundary is a STEP, a wide smoothing window turns it into a ramp, and the gain then ramps with it instead of cancelling it (unit-tested: a 15% step came out 13.5%). Driving every frame to a single number removes steps by construction, because every frame lands on the same number. * LUMA ONLY. Texture drift cannot be fixed here. Blur is the only lever available after the fact, and blur removes real detail along with the model's invented detail - it satisfies a metric by destroying information. A ratcheting chain wants a shorter chain, an anchor-only conditioning diet, or a model-side fix; it does not want a filter. A short median tracks the actual level rather than reacting to single-frame noise. Gain is clamped so nothing is invented and black stays black. """ import torch if mode == "off" or not parts: return parts, "" n = sum(int(p.shape[0]) for p in parts) if n < 8: return parts, "" # Per-frame statistics only - never the whole timeline as one tensor. # Concatenating it costs frames x H x W x 3 x 4 bytes, which is 31 GB for a # 12-shot 1088x1920 chain, and it used to be built three times over. Every # number below this point is one-dimensional. luma = torch.cat([p.mean(dim=(1, 2, 3)) for p in parts]) k = max(3, min(int(med) | 1, (n // 2) * 2 - 1)) pad = k // 2 def _med(v): return torch.nn.functional.pad( v[None, None], (pad, pad), mode="replicate" )[0, 0].unfold(0, k, 1).median(-1).values med_l = _med(luma) target = luma.median() gain = (target / med_l.clamp_min(1e-4)).clamp(0.70, 1.43) if mode == "luma+contrast": # Second drift, measured on master_00014_: the mean holds flat while # the SPREAD grows every hop (p25 6->2, p95 85->96 over three shots). # Matching the mean re-centres that and hands the next shot a # higher-contrast start, so the luma fix was masking it. # # An affine remap about each frame's own mean rescales amplitude and # leaves every edge where it was - not the blur the note below rules # out. Spatial accretion is still not fixable here. sd = torch.cat([p.std(dim=(1, 2, 3)) for p in parts]) med_s = _med(sd) # Anchor to SHOT 1, not the timeline median. Contrast here only ever # ratchets up, so shot 1 is the one frame-set with nothing accreted # onto it - the median target pulls shot 1 UP to meet the drift # (measured on master_00014_: +11.7% texture on shot 1 for no benefit, # 1.069 vs 1.064 per hop). Anchoring to shot 1 leaves it untouched # (+0.1%) and only ever pulls later shots down. s_target = sd[:parts[0].shape[0]].median() cgain = (s_target / med_s.clamp_min(1e-4)).clamp(0.70, 1.43) else: cgain = None # Apply per part, releasing each input as it is consumed. The caller does # `parts, msg = _mn_normalize(parts, ...)` and rebinds immediately, so # dropping the reference here lets each input shot be freed while the rest # of the chain is still being processed. Peak becomes one finished timeline # plus one shot, instead of two whole timelines. res, i = [], 0 for idx in range(len(parts)): p = parts[idx] j = i + int(p.shape[0]) g = gain[i:j, None, None, None] # Frames may be stored fp16 to halve the host timeline. fp16's step near # 1.0 is ~9.8e-4 against an 8-bit output step of 3.9e-3 - enough, but # only 4x. Do the affine in fp32 and store back in the input dtype, so # the memory saving costs no precision. _dt = p.dtype _pf = p.float() if _dt != torch.float32 else p if cgain is not None: m = luma[i:j, None, None, None] _out = ((_pf - m) * cgain[i:j, None, None, None] + m * g).clamp(0, 1) else: _out = (_pf * g).clamp(0, 1) res.append(_out.to(_dt) if _dt != torch.float32 else _out) if _pf is not p: del _pf del _out parts[idx] = None del p i = j after = torch.cat([q.mean(dim=(1, 2, 3)) for q in res]) msg = ("luma %.3f-%.3f -> %.3f-%.3f (target %.3f, gain %.3f-%.3f)" % (float(luma.min()), float(luma.max()), float(after.min()), float(after.max()), float(target), float(gain.min()), float(gain.max()))) if mode == "luma+contrast": a_sd = torch.cat([q.std(dim=(1, 2, 3)) for q in res]) sd0 = sd msg += ("; contrast %.4f-%.4f -> %.4f-%.4f" % (float(sd0.min()), float(sd0.max()), float(a_sd.min()), float(a_sd.max()))) return res, msg def _at_ltas(wav, sr=32000, nfft=2048): """Long-term average spectrum of a [C, L] / [1, C, L] waveform in _AT_NBANDS log-spaced bands (100 Hz .. ~12 kHz). Long-term, so per-shot speech content averages out and only the spectral TILT is measured.""" import torch x = wav.reshape(-1, wav.shape[-1]).float().mean(0) n = (x.shape[-1] // nfft) * nfft if n < nfft: return None S = torch.stft(x[:n], nfft, hop_length=nfft // 2, window=torch.hann_window(nfft), return_complex=True) P = (S.abs() ** 2).mean(-1) f = torch.linspace(0, sr / 2, P.shape[0]) edges = torch.logspace(2, 4.08, _AT_NBANDS + 1) out = [] for i in range(_AT_NBANDS): m = (f >= edges[i]) & (f < edges[i + 1]) out.append(P[m].mean() if m.any() else P.new_tensor(0.0)) return torch.stack(out).clamp_min(1e-12) def _at_flatten(wav, house, sr=32000, nfft=2048, max_db=9.0): """EQ-match a shot's long-term spectral envelope to the house envelope. The audio twin of _cg_flatten, aimed the other way: chained audio drifts DULLER per hop where chained video drifts sharper. Measured 2026-08-11 on 8-shot chains: 4-10 kHz energy fell 84-92% on pure-recency conditioning (bank_pinned=0) and 8-13% even with the pinned slot. Constant per-shot gains applied via STFT - a linear filter, so no pumping. Clamped to +/-max_db, half-strength in the top band so it cannot manufacture hiss where the model genuinely rendered none. """ import torch cur = _at_ltas(wav, sr, nfft) if cur is None or house is None: return wav, 0.0 gain_db = (10.0 * torch.log10(house / cur)).clamp(-max_db, max_db) gain_db[-1] = gain_db[-1] * 0.5 if float(gain_db.abs().max()) < 0.75: return wav, 0.0 f = torch.linspace(0, sr / 2, nfft // 2 + 1) edges = torch.logspace(2, 4.08, _AT_NBANDS + 1) centres = (edges[:-1] * edges[1:]).sqrt() logf = torch.log10(f.clamp_min(1.0)) logc = torch.log10(centres) idx = torch.bucketize(logf, logc).clamp(1, _AT_NBANDS - 1) x0, x1 = logc[idx - 1], logc[idx] w = ((logf - x0) / (x1 - x0)).clamp(0, 1) curve = 10.0 ** ((gain_db[idx - 1] * (1 - w) + gain_db[idx] * w) / 20.0) shape = wav.shape x = wav.reshape(-1, shape[-1]).float() win = torch.hann_window(nfft) S = torch.stft(x, nfft, hop_length=nfft // 4, window=win, return_complex=True) S = S * curve.unsqueeze(0).unsqueeze(-1) y = torch.istft(S, nfft, hop_length=nfft // 4, window=win, length=shape[-1]) return y.reshape(shape).to(wav.dtype), float(gain_db.abs().max()) def _mmh3_encode_ref_audio(audio_vae, audio): """ComfyUI's ref-audio encoder, wherever this ComfyUI keeps it. Up to 0.33 it was a staticmethod on MiniMaxH3ReferenceToVideo; master (2026-08) moved it to a module-level _encode_ref_audio. Resolve at call time so neither layout raises AttributeError (GitHub issue #15).""" from comfy_extras import nodes_minimax_h3 as mmh3 fn = getattr(mmh3, "_encode_ref_audio", None) if fn is None: fn = getattr(getattr(mmh3, "MiniMaxH3ReferenceToVideo", None), "_encode_ref_audio", None) if fn is None: raise RuntimeError( "This ComfyUI's nodes_minimax_h3 has no _encode_ref_audio " "(neither module-level nor on MiniMaxH3ReferenceToVideo). " "Update ComfyUI-H3-Multishot, or report the ComfyUI version.") return fn(audio_vae, audio) def _wav_for_vae(audio_vae, audio, what): """AUDIO dict -> [1, C, L] waveform at the VAE's own sample rate, stereo. Mirrors the native node's _encode_ref_audio (nodes_minimax_h3.py): resample to the VAE's rate before encoding. The spine path used to skip this, so a 44.1/48 kHz voice file - i.e. nearly every real-world file - was encoded as if it were 32 kHz. The latent was garbage, and because the spine LOCKS the audio stream to that latent at every sampling step, the render came out as static (user-reported on ref2va; the same file worked through the native node, which resamples). Mono is upmixed to stereo by duplication - the encoder wants two channels, and refusing a mono file helps nobody. """ w = audio["waveform"] sr = int(audio["sample_rate"]) w3 = w if w.ndim == 3 else w.unsqueeze(0) vae_sr = getattr(audio_vae, "audio_sample_rate", 32000) if sr != vae_sr: import torchaudio w3 = torchaudio.functional.resample(w3, sr, vae_sr) print(f"[H3] {what}: resampled {sr} -> {vae_sr} Hz", flush=True) if w3.shape[1] == 1: w3 = w3.repeat(1, 2, 1) print(f"[H3] {what}: mono upmixed to stereo", flush=True) return w3[:1], vae_sr def _write_shot_mp4(imgs, wav, sr, prefix, label, tag): """Write one decoded shot to disk immediately, and never let that kill the render. A long chain represents hours of GPU time that only becomes a file at the very end, when the master is muxed. Anything that fails after the last shot - a mux OOM, a full disk, a cancelled tab - has historically destroyed every shot at once (issue #13). Each shot written as it decodes turns that from lost work into a joining job. Returns the path written, or None (already reported) on failure. """ try: import os from fractions import Fraction import folder_paths from comfy_api.latest import InputImpl, Types w = wav if wav.ndim == 3 else wav.unsqueeze(0) folder, fname, counter, _sub, _pfx = folder_paths.get_save_image_path( prefix, folder_paths.get_output_directory(), imgs.shape[2], imgs.shape[1]) path = os.path.join(folder, f"{fname}_{counter:05}_.mp4") InputImpl.VideoFromComponents(Types.VideoComponents( images=imgs.detach().cpu(), audio={"waveform": w.detach().cpu(), "sample_rate": sr}, frame_rate=Fraction(24))).save_to(path) print(f"[{tag}] {label} -> {path}", flush=True) return path except Exception as e: print(f"[{tag}] {label} FAILED to save (render continues): {e}", flush=True) return None def _up_model_factor(model, default=4.0): """The fixed enlargement factor of a loaded upscale model. Needed to predict output size before anything is upscaled. ESRGAN-family models expose it as `scale`; fall back to 4x, which is what almost every model in circulation is, rather than refusing to predict. """ for attr in ("scale", "scale_factor", "upscale_factor"): v = getattr(model, attr, None) if v is None: v = getattr(getattr(model, "model", None), attr, None) try: if v and float(v) > 0: return float(v) except (TypeError, ValueError): pass return default def _upscale_frames(imgs, scale, model, tag): """Enlarge decoded frames, AFTER sampling. Pixels, never latents. The old two-pass path interpolated the raw latent between passes; H3's latent is not spatially smooth, so that landed off-manifold and produced colour noise no matter how the schedule was split. Anything done here is downstream of the VAE, so it cannot leave the manifold - the worst case is a soft picture, not a broken one. Per shot, so peak memory is one shot's frames rather than the whole chain. """ if model is None and (not scale or abs(scale - 1.0) < 1e-6): return imgs import comfy.utils h, w = int(imgs.shape[1]), int(imgs.shape[2]) if model is not None: from comfy_extras.nodes_upscale_model import ImageUpscaleWithModel import comfy.model_management as _mm import os as _os import torch as _t # Chunked. One call with every frame allocates the whole upscaled batch # in fp32 on the CPU on top of the input - 11 GB for 243 frames at # 1472x2560, which is where a 12-shot 736x1280 run died after two hours. # Writing into a preallocated output holds (output + one chunk) instead. _n = int(imgs.shape[0]) _ch = max(1, int(_os.environ.get("H3_UPSCALE_CHUNK", "16"))) if _n > _ch: _out = None for _s in range(0, _n, _ch): _p = ImageUpscaleWithModel().upscale(model, imgs[_s:_s + _ch])[0] if _out is None: _out = _t.empty((_n,) + tuple(_p.shape[1:]), dtype=_p.dtype) _out[_s:_s + int(_p.shape[0])] = _p del _p imgs = _out else: imgs = ImageUpscaleWithModel().upscale(model, imgs)[0] # Free it AT ONCE. Left resident it survives into the next shot, and # the DiT can then only load partially - measured 18.5 s/it on shot 1 # (full load) against 349 s/it on shot 2 (431 MB offloaded), a 19x # collapse from ~65 MB of upscaler weights holding the door open. try: _mm.free_memory(_mm.get_total_memory(_mm.get_torch_device()), _mm.get_torch_device(), [getattr(model, "patcher", None)]) if hasattr(model, "patcher"): model.patcher.model.to(_mm.unet_offload_device()) _mm.soft_empty_cache() except Exception as _e: print("[%s] upscale: could not free the upscaler (%s) - the next " "shot may load the DiT partially and run far slower" % (tag, _e), flush=True) print("[%s] upscale: model %dx%d -> %dx%d" % (tag, w, h, int(imgs.shape[2]), int(imgs.shape[1])), flush=True) if scale and abs(scale - 1.0) > 1e-6: # the model has a fixed factor; land on the size actually asked for th, tw = int(round(h * scale)), int(round(w * scale)) x = imgs.movedim(-1, 1) x = comfy.utils.common_upscale(x, tw, th, "lanczos", "disabled") imgs = x.movedim(1, -1) print("[%s] upscale: resized to %dx%d" % (tag, tw, th), flush=True) return imgs th, tw = int(round(h * scale)), int(round(w * scale)) x = imgs.movedim(-1, 1) x = comfy.utils.common_upscale(x, tw, th, "lanczos", "disabled") print("[%s] upscale: lanczos %dx%d -> %dx%d" % (tag, w, h, tw, th), flush=True) return x.movedim(1, -1) def _smart_head_trim(wav, sr, trim, search_s=0.75): """Remove `trim` samples from a chained shot's audio HEAD, cutting at the QUIETEST spot in the first `search_s` seconds instead of blindly at sample 0. The blind head cut clips the attack of any word the model placed at the very start of the shot (the 'blip' - ear-verified on a render 2026-08-10: a syllable burst 0.4s after the weld with its front shaved). Cutting the same number of samples out of the quietest window preserves every onset; the audio before the cut plays at most trim/sr (~42ms) late against the video, in a region that is quiet by construction, and sync is exact after the cut. Quietest-window-at-0 reproduces the old behaviour bit for bit.""" import torch n = wav.shape[-1] if n <= trim: return wav[..., :0] limit = min(n - trim, int(sr * search_s)) if limit <= 1: return wav[..., trim:] mono = wav.float().abs() while mono.ndim > 1: mono = mono.mean(0) sq = mono[:limit + trim] ** 2 cs = torch.cumsum(torch.cat([torch.zeros(1, device=sq.device), sq]), 0) win_energy = cs[trim:limit + trim] - cs[:limit] i = int(win_energy.argmin()) if i > 0: print(f"[H3Multishot] smart weld: seam cut moved {i / sr * 1000:.0f}ms " f"into the head (quietest gap), word onsets preserved", flush=True) return torch.cat([wav[..., :i], wav[..., i + trim:]], dim=-1) def _xfade_audio(parts, sr, ms=40): """Concatenate shot audio with a short equal-power crossfade at each seam. Each shot is sampled independently, so its waveform starts and ends at a hard boundary. Butt-joining them puts a step discontinuity in the signal at every seam, which reads as a click and as "spliced clips" to a listener. A ~40ms equal-power fade removes the step without audibly shortening anything. """ import torch if not parts: return None if len(parts) == 1: return parts[0] n = max(1, int(sr * ms / 1000.0)) out = parts[0] for nxt in parts[1:]: k = min(n, out.shape[-1], nxt.shape[-1]) if k < 8: # too short to fade; butt-join out = torch.cat([out, nxt], dim=-1) continue t = torch.linspace(0, 1, k, dtype=out.dtype, device=out.device) fade_out = torch.cos(t * 3.14159265 / 2) # equal power fade_in = torch.sin(t * 3.14159265 / 2) head, tail = out[..., :-k], out[..., -k:] seam = tail * fade_out + nxt[..., :k] * fade_in out = torch.cat([head, seam, nxt[..., k:]], dim=-1) return out def _parse_script(text): """JoyEcho script -> list of shot prompts. JSON {"prompts": [...]} or plain text with --- separators. Malformed JSON fails LOUD.""" text = (text or "").strip() shots = [] if text.startswith("{") or text.startswith("["): data, repaired = _repair_json(text) if data is None: raise ValueError( f"H3 script looks like JSON but does not parse ({repaired}). " f"Auto-repair of unclosed brackets/quotes was attempted and " f"failed. Common cause: a doubled {{ on the first lines, or a " f"missing comma between prompts. Fix the script or use plain " f"prompts separated by --- lines.") if repaired: print(f"[H3Multishot] script JSON was incomplete; auto-repaired " f"({repaired}). Consider fixing the source.", flush=True) if isinstance(data, dict): shots = [str(p) for p in data.get("prompts", [])] elif isinstance(data, list): shots = [str(p) for p in data] if not shots: shots = [b.strip().replace('\\"', '"') for b in re.split(r"(?m)^---\s*$", text) if b.strip()] if not shots: shots = [text] return shots class H3ScriptSplit: @classmethod def INPUT_TYPES(cls): return {"required": {"script": ("STRING", { "multiline": True, "dynamicPrompts": False, "default": "Shot 1 prompt goes here.\n---\n" "Shot 2 prompt goes here.\n---\n" "Shot 3 prompt goes here.", "tooltip": "One prompt per shot, separated by --- on its own " "line. (JSON {\"prompts\": [...]} also accepted, for " "generated scripts.)"}), "shot_count": ("INT", { "default": 0, "min": 0, "max": 3, "tooltip": "This workflow ALWAYS renders 3 segments and " "joins them (~30s master). 0 = count from the " "script. 3 = three scenes. 2 = the third segment " "continues scene 2. 1 = one scene sustained for " "the full 30s. Scripts with >3 prompts: extras " "are dropped (see console)."}), }} RETURN_TYPES = ("STRING", "STRING", "STRING", "STRING", "INT") RETURN_NAMES = ("shot_1", "shot_2", "shot_3", "shot_4", "shot_count") FUNCTION = "split" CATEGORY = "conditioning/minimax" def split(self, script, shot_count=0): shots = _parse_script(script) if shot_count and shot_count > 0: if len(shots) > shot_count: print(f"[H3ScriptSplit] shot_count={shot_count}: dropping " f"{len(shots) - shot_count} extra script shot(s).", flush=True) shots = shots[:shot_count] while len(shots) < shot_count: shots.append(shots[-1]) n = len(shots) if n < 3: print(f"[H3ScriptSplit] script has {n} shot(s); a 3-shot graph " f"will render the last prompt {3 - n} extra time(s) as a " f"continuation.", flush=True) elif n > 3: print(f"[H3ScriptSplit] script has {n} shots; a 3-shot graph " f"DROPS shot(s) 4+. Trim the script or wait for the " f"dynamic-count workflow.", flush=True) while len(shots) < 4: shots.append(shots[-1]) return (shots[0], shots[1], shots[2], shots[3], n) # --------------------------------------------------------------------------- # AUTO ACTIVATION RESERVE # # VRAM has two tenants with opposite tolerance for being remote. WEIGHTS # stream well: sequential, known order, prefetched behind ~50s of compute per # step, so 20GB offloaded costs well under a second. The ALLOCATOR POOL # (activations) does not stream: it is random-access and re-touched all step, # and when it does not fit, the driver evicts blind mid-step. Measured on a # 3090: starving the pool by ~3GB = 533 s/it vs 99 s/it. Measured on a 5090: # 168W at "99% utilisation" - a card waiting, not computing. # # So the correct split is reserve >= pool, and stream whatever weights do not # fit - overshooting is nearly free, undershooting is 5-10x. The pool scales # with the render shape, which is why any hand-set number (a GB figure, a # memory_usage_factor) is correct for exactly one resolution and a trap at # every other: LOWER the resolution with a fixed factor and the reserve # shrinks below the pool - the render gets SLOWER, the opposite of what any # person expects. # # This engine removes the knob: # - memory_required(input_shape) is overridden with a function of the # ACTUAL shape comfy passes at load time - never a constant. # - Unmeasured shapes reserve 60% of currently-free VRAM: generous enough # to be cliff-proof at any resolution, and only slightly slower than # optimal (a few more GB of weights stream). # - Our samplers measure the true allocator peak of every run and cache it # per (GPU, model, shape-cells) in the user dir. From the second run at # a shape, the reserve is measured * 1.25 - per machine, no telemetry to # read, no number to know. # --------------------------------------------------------------------------- _AUTO_FLOOR = 8 * 1024**3 # never reserve less: workspaces + margin # First-run fraction of free VRAM. Deliberately HIGH: over-reserving merely # streams more weights (<1s/step behind 50-100s of compute), while # under-reserving is the 5-10x cliff. 0.60 was calibrated on a 32GB card and # proved WRONG on a 24GB one: 60% of the 3090's 21.9GB free = 13.2GB against # a ~17.5GB pool -> max_reserved 25.06GB on a 24GB card -> 492 s/it. At 0.88 # the same card reserves 19.3GB and streams the difference. Measurement then # tightens DOWN from the safe side. _AUTO_FRACTION = 0.88 # unmeasured shapes: fraction of free VRAM _AUTO_WEIGHT_NUCLEUS = 2 * 1024**3 # always leave a little room for weights _AUTO_MARGIN = 1.25 # measured pool -> reserve headroom _AUTO_KEEPOUT = 1024 * 1024**2 # left free beyond weights+pool. 384 MB was # too small: ComfyUI reserves its own ~117 MB # buffer and counts 'usable' differently from # get_free_memory, so a clamp computed to keep # 20.3 GB of weights resident still offloaded # 401 MB and then aborted in a CUDA kernel. _AUTO_MIN_POOL = 1536 * 1024**2 # below this, prefer a loud OOM to a silent crawl _auto_cache = None # lazy {key: pool_bytes} _auto_last = {"key": None, "model": None} # what the next sampling run is _auto_session = {} # key -> reserve pinned for this session def _auto_cache_path(): try: import folder_paths base = folder_paths.get_user_directory() except Exception: import os base = os.path.dirname(os.path.abspath(__file__)) import os return os.path.join(base, "h3_auto_reserve.json") _AUTO_SCHEMA = 2 # bumped when the pool measurement changed meaning def _auto_cache_load(): global _auto_cache if _auto_cache is None: import io as _io, os _auto_cache = {} p = _auto_cache_path() if os.path.isfile(p): try: _auto_cache = json.load(_io.open(p, encoding="utf-8")) except Exception: _auto_cache = {} # Entries written before the full-weights fix can be too large by # however much the DiT had been offloaded that shot, and the store # keeps the largest value forever - so one bad shot poisons a shape # permanently. Drop pre-schema caches once instead of carrying them. if _auto_cache.pop("_schema", None) != _AUTO_SCHEMA: _stale = len(_auto_cache) if _stale: print("[H3AutoReserve] discarded %d cached reserve(s) written " "before the partial-load measurement fix. Those numbers " "could be inflated by the offloaded weight bytes and " "never shrank. They re-measure on the next run." % _stale, flush=True) _auto_cache = {} return _auto_cache def _auto_cache_store(key, pool_bytes): """Record a pool measurement. Returns True if it raised the stored value.""" cache = _auto_cache_load() prev = cache.get(key, 0) # keep the largest pool ever seen for the shape; shrinking on a lucky # run risks the cliff on the next unlucky one if pool_bytes <= prev: return False cache[key] = int(pool_bytes) try: import io as _io _out = dict(cache) _out["_schema"] = _AUTO_SCHEMA _io.open(_auto_cache_path(), "w", encoding="utf-8").write( json.dumps(_out, indent=1)) except Exception as e: print(f"[H3AutoReserve] cache write failed ({e}) - measurements " f"will not persist across restarts", flush=True) return True def _auto_cache_floor(key, pool_bytes): """Record a LOWER BOUND from a run whose peak was truncated by the card. Same ratchet as _auto_cache_store - it only ever raises - but named separately at the call site so it is obvious that the number is known to understate the real need. Without this a ceiling-bound shot contributes nothing at all, which is how the chained shots deadlocked: their payload key stayed empty, the fallback reserve was too small for them, and every attempt to measure was thrown away for being too small. """ return _auto_cache_store(key, pool_bytes) _auto_payload = {"sig": ""} def _auto_set_payload(sig): """Samplers call this before each shot with a conditioning-payload signature (keyframes / reference blocks / audio refs). Two shots with the same latent shape but different payloads need DIFFERENT pools - render-verified 2026-08-10: shot 1 (bare) measured a 4.9 GB pool, shot 2 (chain keyframe + 10s self-anchor audio ref) then spilled to system RAM at that reserve and ran 5x slower per step.""" _auto_payload["sig"] = str(sig or "") def _payload_scheme(sig): """Which sampler wrote a payload signature: H3MultishotSampler writes "kf%da%d...", H3MultishotMemorySampler writes "%s%d_k%dr%d...". The two namespaces never compare equal, so a raw string mismatch is NOT evidence of a heavier payload (24 GB test-lab finding F008, 2026-08-16).""" return "kf" if str(sig or "").startswith("kf") else "cont" def _payload_mult(src_sig, dst_sig): """Bare->payload scale when borrowing across signatures. Same string or different sampler scheme -> 1.0 (equal-cells evidence taken as-is); within one scheme the measured x1.6 bare->payload jump stands. The old flat "1.6 unless identical" fired on every CORE run against a Memory-sampler cache: 12.4 GB became a 19.8 GB request, 129 MB of DiT stayed resident, 66 s/it (24 GB lab, S1).""" if src_sig == dst_sig: return 1.0 if _payload_scheme(src_sig) != _payload_scheme(dst_sig): return 1.0 return 1.6 _PAYLOAD_ADD_BYTES = 4 * 1024**3 # a keyframe + an audio ref, in GB, roughly fixed def _payload_need(bare, src_sig, dst_sig): """Pool NEED for a payload signature, from a bare (or other) measurement. 24 GB test-lab finding F010 (2026-08-16/17): the flat x1.6 was above the 93rd percentile of 16 measured bare->payload pairs (median 1.40x, max 1.73x) and the cost is ADDITIVE, not multiplicative - a largely fixed number of GB, so a big relative jump on a 6 GB pool and negligible on a 15 GB one. Measured: 6 GB bare -> ~1.5x; 11 GB -> +3.1..+4.0 GB; 15 GB -> ~1.1x. A flat multiplier therefore over-reserved worst at large geometry - exactly where a 24 GB card evicts weights instead (S3 run 3 reserved 20.6 GB for a 15.4 GB need). Model it as +4 GB capped at the old x1.6, so it can only ask for less than before, never more. """ m = _payload_mult(src_sig, dst_sig) if m <= 1.0: return int(bare) return int(min(bare * 1.6, bare + _PAYLOAD_ADD_BYTES)) def _model_family(stem): """Coarse quant family for the reserve borrow. The pool tracks geometry WITHIN a family, not across: comfy-native quant paths (w4a8/int8/nvfp4) carry dequant scratch that GGUF does not - the same card's cache rates read 3.0-5.7 GB/Mcell for w4a8 against 2.2-2.5 for GGUF, and a GGUF- based borrow under-reserved a w4a8 first run into WDDM paging (the 24 GB lab box lab F009, 2026-08-16). Under-reserving is the fatal direction.""" s = str(stem or "").lower() if re.search(r"q\d|gguf|curve|k_[msl]|_q[0-9]", s): return "gguf" return "native" _CROSS_FAMILY_MULT = 1.8 # w4a8/gguf per-cell ratio measured on the 3090 cache def _auto_key(model_name, cells, sig=None): try: import torch dev = torch.cuda.get_device_name(0) except Exception: dev = "cpu" import os stem = os.path.splitext(os.path.basename(model_name))[0] s = _auto_payload["sig"] if sig is None else sig return f"{dev}|{stem}|{cells}|{s}" def _install_auto_reserve(patcher, model_name): """Shape-aware memory_required on the BaseModel (clone-safe).""" def memory_required(input_shape, *args, **kwargs): cells = 1 try: for d in list(input_shape)[1:]: cells *= int(d) except Exception: cells = 0 # Fingerprint the patch set into the shape key: a chunked-FFN or # sol-attn model has a different real pool than the same shape # unpatched, and the largest-ever cache erased chunking's benefit # (2026-08-23: chunk_ffn ON planned against the unchunked 9.8 GB # measurement and streamed weights for nothing). Folded into the # NAME segment so key.rsplit("|") structure is unchanged. try: _po_fp = getattr(patcher, "model_options", {}) or {} _fp = "~fp%d.%d" % ( len(_po_fp.get("patches", {}) or {}), len(_po_fp.get("transformer_options", {}) or {})) except Exception: _fp = "" key = _auto_key(model_name + _fp, cells) _auto_last["key"] = key # comfy (and DynamicVRAM) call memory_required repeatedly - per load # AND per sampling step. The answer must be STABLE for a shape: # recomputing "60% of free" as free shrinks is a feedback loop # (reserving memory reduces free, which reduces the next answer). # Pin the first computation per (model, shape) for the session; # a fresh measurement invalidates the pin. pinned = _auto_session.get(key) if pinned is not None: return pinned cache = _auto_cache_load() measured = cache.get(key) or 0 _known_need = int(measured) # best evidence of true pool need if measured: # a real measurement carries its own x1.25 margin - the old # 8 GB floor here overrode good small measurements and forced # weight offload on 24 GB cards for nothing reserve = max(int(measured * _AUTO_MARGIN), 2 * 1024**3) how = f"measured pool {measured/2**30:.1f} GB x {_AUTO_MARGIN}" else: # unmeasured payload variant of a measured shape: estimate from # the sibling instead of guessing from free VRAM. Reference and # keyframe tokens ride every step; x1.6 covered the measured # bare->payload jump with margin. sib = sib_sig = None prefix = key.rsplit("|", 1)[0] + "|" mysig = key.rsplit("|", 1)[1] for k2, v2 in cache.items(): if k2.startswith(prefix) and v2 and (sib is None or v2 > sib): sib, sib_sig = v2, k2.rsplit("|", 1)[1] if sib: _known_need = _payload_need(sib, sib_sig, mysig) reserve = max(int(_known_need * _AUTO_MARGIN), _AUTO_FLOOR) how = (f"payload variant of a measured shape: sibling pool " f"{sib/2**30:.1f} GB -> need {_known_need/2**30:.1f} GB " f"(payload +4 GB capped x1.6) x {_AUTO_MARGIN}") else: # FIRST RUN at an unseen (model, shape). The pool tracks the # GEOMETRY, not the checkpoint - measured 2026-08-16: a Q8 # GGUF and a mixed-precision file wanted the same ~11-13 GB # at the same cells. So borrow the nearest measurement on # this card from ANY model and shape, scaled by cell count, # before falling back to the free-VRAM placeholder - which # planned 3.3 GB for an 11 GB pool twice in one afternoon # and produced a crawl on a 5090 and a dead CUDA context on # a 3090. Max over candidates: over-reserving streams a few # GB of weights (cheap); under-reserving pages (fatal). borrowed = bfrom = None try: mydev, _, _, mysig = key.split("|", 3) # same-payload siblings first; only if none exist, other # payloads with the bare->payload x1.6 on top. Otherwise a # bare shot 1 borrows a pinned shot's pool times 1.6 and # over-reserves by double. # nearest shape wins, NOT the largest estimate: pools are # not purely linear in cells (fixed overhead fattens the # per-cell rate of small shapes), so extrapolating far # overshoots - a real cache here scaled small w4a8 shapes # to 26 GB while two same-cells entries said 14-15. # rank: shape distance first, then same payload, then the # larger estimate. A same-cells measurement beats a # same-payload one from a distant shape - a real cache # extrapolated distant small shapes to 26-31 GB while # same-cells entries said 15. best = None myfam = _model_family(key.split("|", 3)[1]) for k2, v2 in cache.items(): try: d2, m2, c2, s2 = k2.split("|", 3) c2 = int(c2) except ValueError: continue if d2 != mydev or not v2 or not c2 or not cells: continue ratio = cells / c2 if not (0.4 <= ratio <= 2.5): continue # no wild-scale extrapolation fam_miss = 0 if _model_family(m2) == myfam else 1 est = int(_payload_need(v2 * ratio, s2, mysig) * (_CROSS_FAMILY_MULT if fam_miss else 1.0)) # same quant family first (F009), then nearest shape, # then same payload, then the fatter estimate rank = (fam_miss, abs(ratio - 1.0), 0 if s2 == mysig else 1, -est) if best is None or rank < best: best, borrowed, bfrom = rank, est, (c2, v2) except Exception: borrowed = None if borrowed: _known_need = borrowed reserve = max(int(borrowed * _AUTO_MARGIN), _AUTO_FLOOR) how = ("borrowed pool: %.1f GB measured at cells=%d " "scaled to %.1f GB (pool tracks geometry, not " "the checkpoint)" % (bfrom[1] / 2**30, bfrom[0], borrowed / 2**30)) print("[H3AutoReserve] first run with this model at this " "shape - borrowing a measured pool from another " "model/shape on this card: %.1f GB at cells=%d " "scales to %.1f GB here. This shot still records " "its own measurement." % (bfrom[1] / 2**30, bfrom[0], borrowed / 2**30), flush=True) else: try: import comfy.model_management as mm free = mm.get_free_memory(mm.get_torch_device()) except Exception: free = 24 * 1024**3 reserve = max(int(min(free * _AUTO_FRACTION, free - _AUTO_WEIGHT_NUCLEUS)), _AUTO_FLOOR) how = (f"first run at this shape: " f"{_AUTO_FRACTION:.0%} of free") # CLAMP AGAINST THE CARD. Every GB reserved here comes out of the # weights budget, and a DiT that misses a FULL load streams the # remainder over PCIe every step. Measured 2026-08-12 at 960x544: # shot 1 reserved 7.8 GB and loaded completely at 18.8 s/it; shot 2's # larger payload reserved 9.4 GB, left the DiT 399 MB short, and ran # at 283 s/it - a 15x collapse bought by 1.6 GB of headroom the # measurement said was not needed. The failure modes are asymmetric: # too small OOMs loudly and you fix it, too large silently costs 15x. # So the pool yields to the weights - but ONLY while the cut stays at # or above the measured need. Cutting below it does not keep the # weights resident (the allocator evicts them for real allocations # regardless - measured on a 3090: clamped to 1.5 GB "to keep weights # resident" and the shot's own measurement then read resident 0.0), # so past that line the weights yield instead. try: import comfy.model_management as _cm _dev = _cm.get_torch_device() _free = _cm.get_free_memory(_dev) _w = int(patcher.model_size()) _cap = int(_free - _w - _AUTO_KEEPOUT) if _cap <= 0: # Before accepting the tight regime, sweep leftovers: an item # that died without a clean boundary leaves the previous # model resident, and only the multishot remote-TE lane had a # shot-1 sweep - the classic/local-TE lane planned against # the stale number and paged at 141 W (twice, 2026-08-22). # Everything resident here has finished its work (the TE # encodes before the DiT plans), so unloading costs one # reload at worst; ComfyUI reloads on demand. try: _before_sw = _free _cm.unload_all_models() _cm.free_memory(_cm.get_total_memory(_dev) * 0.9, _dev) _free = _cm.get_free_memory(_dev) _cap = int(_free - _w - _AUTO_KEEPOUT) if _free - _before_sw > 2**30: print("[H3AutoReserve] cleared %.1f GB of leftovers " "before reserve planning (reported free " "%.1f -> %.1f GB)." % ((_free - _before_sw) / 2**30, _before_sw / 2**30, _free / 2**30), flush=True) except Exception: pass if _cap <= 0: # The weights alone exceed free VRAM. Reserving the measured # pool here is the WORST possible move: every byte of reserve # pushes another byte of weights out, and the old `_cap > 0` # guard skipped the clamp entirely in exactly this case. A # 3090 with 10.3 GB free and 20.3 GB of weights reserved # 22.1 GB and loaded "0.00 MB usable, 0.00 MB loaded, # 20796.43 MB offloaded" - every layer streamed off disk on # every step, 128 s/it against ~29 s/it resident, and after # four hours the file-reader gave out with # hostbuf_file_reader_read failed. _was = reserve # NOT the floor. Reserving the minimum guarantees the weights # load but starves the activation pool, and the allocator then # spills ACTIVATIONS instead - measured worse than the problem # it replaced: 36 -> 55 -> 331 s/it across three shots while # every shot still reported 'loaded completely, full load: # True'. Use the measured pool when a sibling shot has given # us one: shot 2 measured 8.1 GB and peaked at 22.6 with 14.5 # GB of weights, which fits a 24 GB card exactly. The inflated # payload estimate (x1.6 x1.25 compounding off an already # bumped sibling) is what asks for 16 GB and does not fit. reserve = (max(int(_known_need), _AUTO_MIN_POOL) if _known_need else _AUTO_MIN_POOL) reserve = max(min(reserve, int(_free - _AUTO_KEEPOUT)), _AUTO_MIN_POOL) how += (" | TIGHT %.1f -> %.1f GB: weights %.1f GB vs " "%.1f GB reported free" % (_was / 2**30, reserve / 2**30, _w / 2**30, _free / 2**30)) print("[H3AutoReserve] TIGHT: %.1f GB of weights against " "%.1f GB reported free, so this shot has no headroom. " "Reserving %.1f GB (the measured pool) rather than the " "payload estimate, which does not fit. Note the reported " "figure understates what ComfyUI ends up with, so the " "weights may still load completely - watch for a spill " "instead: high GPU utilisation at low wattage. If the " "render crawls, lower frames_per_shot or resolution, free " "the other ComfyUI instance, or load a smaller DiT." % (_w / 2**30, _free / 2**30, reserve / 2**30), flush=True) # Driver headroom applies HERE too. Reported free is misleading # in this branch (resident weights count as used but get # reused), so the peak still lands wherever weights+pool put # it - measured 2026-08-16 on a 5090: the tight requeue of a # shape whose first run peaked at a healthy 29.4/32.6 sat at # 31.5/32.6 and crawled at 142 W. The main headroom block below # is gated on _cap > 0 and its clamp uses reported free, both # wrong for this regime, so bump against TOTAL directly: # streaming ~2 GB more weights is cheap, the last few percent # of VRAM are not. try: _total_t = _cm.get_total_memory(_dev) except Exception: _total_t = _free # No keepout credit: comfy fills weights to free-reserve no # matter what, so crediting keepout under-raised the reserve # by ~1 GB and left the peak in the WDDM zone (767 s/it, # Zara run 2026-08-23). Full bump; 1 GB extra streaming is # the measured-cheap side. _bump = int(_total_t * 0.09) if _bump: reserve += _bump how += (" | +driver headroom (tight) +%.1f GB" % (_bump / 2**30)) print("[H3AutoReserve] driver headroom (tight path): " "raising the reserve by %.1f GB so extra weights " "stream instead of the peak riding the last few " "percent of VRAM (that zone measured 2-12x slower)." % (_bump / 2**30), flush=True) elif reserve > _cap: _was = reserve _card_max = max(int(_free - _AUTO_KEEPOUT), _AUTO_MIN_POOL) if _known_need and _cap < _known_need: # The cut would land below the measured need. Weight # residency is not achievable in this regime - the # allocator evicts weights to satisfy the sampler's real # allocations no matter what is reserved (3090: clamped to # 1.5 GB "to keep weights resident", measurement then read # resident 0.0, 24.3 GB driver spill, ~2x step time). So # give the pool its need, bounded by the card, and let the # weights stream: that is the cheaper side here. # Bare need, not need*margin. The x1.25 margin guards a # pool overrun against PINNED weights; here the weights # stream regardless, so an overrun just evicts a little # more of them - graceful. Every GB of margin trimmed is a # GB of weights that stays resident instead of streaming # every step (measured: 18.4 reserve left 3.3 GB resident, # 14.7 leaves 7.0). reserve = min(max(int(_known_need), _AUTO_MIN_POOL), _card_max) _resid = max(0, int(_free - _AUTO_KEEPOUT) - reserve) how += (" | NEED %.1f -> %.1f GB (bare need, margin " "yielded to weights): ~%.1f GB of weights can stay " "resident" % (_was / 2**30, reserve / 2**30, _resid / 2**30)) print("[H3AutoReserve] pool need %.1f GB cannot fit " "beside %.1f GB of weights in %.1f GB free. " "Reserving %.1f GB for the pool and letting the " "weights stream - clamping the pool here does not " "keep the weights resident, it only adds a driver " "spill on top of the offload." % (_known_need / 2**30, _w / 2**30, _free / 2**30, reserve / 2**30), flush=True) print("[H3AutoReserve] hint: if chunk_ffn is OFF in " "Studio Switches, turning it on roughly halves " "this pool and can keep every weight resident.", flush=True) if _card_max < _known_need: print("[H3AutoReserve] WARNING: even with zero weights " "resident the card has %.1f GB for a %.1f GB " "pool. This can die inside a CUDA kernel. Lower " "frames_per_shot or resolution." % (_card_max / 2**30, _known_need / 2**30), flush=True) else: # No measurement says the cut goes below need, so this is # margin-trimming: keep the weights resident. Measured # 2026-08-12: a 399 MB weight shortfall cost 15x streaming. # BUT the opposite cliff is real too (2026-08-21, 5090): # trimming a BORROWED pool's margin to squeeze 20.4 GB of # weights fully resident left ~0.6 GB of slack where the # streaming path keeps ~1.9, and the peak spilled into # driver memory - 590 s/it against the streaming path's 27 # on the same total budget. The trim gamble is only worth # taking when it is SMALL: if keeping the weights resident # means eating more than 0.75 GB of the pool's margin, # stream a sliver instead - that side of the trade is # measured shallow (1-2 GB streaming ~ 27 s/it all day). # The discriminator is HEADROOM, not trim size: every # clean run keeps ~9% of the card as driver slack, and # both measured crawls ran with less (65 s/it at reduced # slack, 590 s/it at ~0.6 GB). Keep the weights resident # ONLY if the clamped pool still holds the bare need PLUS # the full driver-headroom bump; otherwise stream - the # multi-GB streaming regime is measured shallow (~27 s/it # all day on this card). try: import comfy.model_management as _mmt _tt = _mmt.get_total_memory(_mmt.get_torch_device()) except Exception: _tt = _free _full_bump = int(_tt * 0.09) # no keepout credit (2026-08-23) _clamp_res = max(_cap, _AUTO_MIN_POOL) _bare = int(_known_need) if _known_need else int(reserve / 1.25) if _clamp_res - _bare < _full_bump: reserve = min(_bare + _full_bump, _card_max) how += (" | resident-clamp REFUSED (slack %.1f GB < " "headroom %.1f GB): streaming" % ((_clamp_res - _bare) / 2**30, _full_bump / 2**30)) print("[H3AutoReserve] keeping all %.1f GB of weights " "resident would leave only %.1f GB of slack " "over the pool's bare need - under the %.1f GB " "driver headroom every clean run keeps " "(2026-08-21: that squeeze ran 22x slower than " "streaming). Reserving %.1f GB and letting " "~%.1f GB of weights stream." % (_w / 2**30, (_clamp_res - _bare) / 2**30, _full_bump / 2**30, reserve / 2**30, max(0.0, (_w - (_free - reserve - _AUTO_KEEPOUT))) / 2**30), flush=True) _clamped_resident = False else: reserve = _clamp_res _clamped_resident = True if _clamped_resident: how += (" | CLAMPED %.1f -> %.1f GB to keep the " "weights (%.1f GB) resident out of %.1f GB " "free" % (_was / 2**30, reserve / 2**30, _w / 2**30, _free / 2**30)) # Only shout when the pre-clamp figure came from EVIDENCE. # On a first run at an unseen shape there is no # measurement, and `_was` is the placeholder from the # "%.0f%% of free" branch - a fraction of whatever happens # to be free, not an estimate of this shot's need. It is # therefore always far above the clamp, so this warning # fired on every new shape and predicted a server-killing # crash for renders that were fine. # # Field log 2026-08-15, 5090, two different shapes: # cells=6384960 "asked for 26.5 GB" # cells=6993216 "asked for 26.5 GB" <- 10% more pixels # Identical, because both are 88% of the same 30.1 GB free. # Both clamped, both rendered clean, and both then MEASURED # 11.4 GB - less than half the figure being warned about. # A warning that cries wolf on every new resolution teaches # people to cancel jobs that would have worked. if _was > reserve * 1.35 and _known_need: print("[H3AutoReserve] WARNING: this shape previously " "MEASURED a %.1f GB activation pool and only " "%.1f GB is available after the %.1f GB of " "weights. That is not a slow render - it usually " "dies inside a CUDA kernel and takes the server " "with it. Lower frames_per_shot or resolution, or " "load a smaller quantisation of the DiT." % (_known_need / 2**30, reserve / 2**30, _w / 2**30), flush=True) elif _was > reserve * 1.35: print("[H3AutoReserve] first run at this shape - the " "%.1f GB figure is a placeholder (%.0f%% of " "free), not a measurement, and has been clamped " "to %.1f GB so the %.1f GB of weights stay " "resident. The real pool gets measured during " "this shot and used from the next one on." % (_was / 2**30, _AUTO_FRACTION * 100, reserve / 2**30, _w / 2**30), flush=True) if measured and reserve < measured: how += (" [tight: below the measured pool %.1f GB]" % (measured / 2**30)) # DRIVER HEADROOM (measured 2026-08-16). Five identical runs # at ~96% VRAM took 27-175 minutes - same shape, same seed, a # lottery. The same run with the reserve raised so ~3.6 GB of # weights streamed instead peaked at 29.5/32.6 and took 15 # minutes, faster than every resident run. WDDM demotes # unpredictably in the last few percent of VRAM, and streamed # weights are cheap (core prefetch overlaps them) - so when # weights + pool would land in that zone, RAISE the reserve: # it is the one lever that pushes weights off and peak down. if _cap > 0: try: _total = _cm.get_total_memory(_dev) except Exception: _total = _free _wddm = int(_total * 0.09) _pool_real = int(_known_need) if _known_need else reserve # Full wddm on top of the pool - no keepout credit (see the # tight-path note; the 2026-08-23 767 s/it crawl planned # reserve = pool + wddm - keepout and peaked 1.1 GB into # the zone this exists to keep clear). # FIXED 2026-08-24 (finetooth): the target must sit on top of # the RESERVED pool (which already carries the x1.25 overrun # margin), not the bare measured pool. Comparing reserve # against pool+wddm silently vetoed the raise whenever the # measured pool exceeded 36% of the card - i.e. on every # production shape this guard exists for. _target = max(int(reserve), _pool_real) + _wddm if (_w + _pool_real + _AUTO_KEEPOUT + _wddm > _free and reserve < _target): _target = min(_target, max(int(_free - _AUTO_KEEPOUT), _AUTO_MIN_POOL)) how += (" | +driver headroom %.1f -> %.1f GB" % (reserve / 2**30, _target / 2**30)) print("[H3AutoReserve] driver headroom: raising the " "reserve %.1f -> %.1f GB so ~%.1f GB of weights " "stream instead of riding the last few percent " "of VRAM (that zone measured 2-12x slower)." % (reserve / 2**30, _target / 2**30, max(0.0, (_w + _pool_real + _AUTO_KEEPOUT + _wddm - _free)) / 2**30), flush=True) reserve = _target except Exception as _hr_e: # never silent: 275 lines of clamp/headroom logic funneled into a # bare pass meant any error here dropped the raise with no trace print("[H3AutoReserve] clamp/headroom pass skipped on error: %s" % _hr_e, flush=True) _auto_session[key] = reserve print(f"[H3AutoReserve] shape cells={cells}: reserving " f"{reserve/2**30:.1f} GB ({how})", flush=True) return reserve patcher.model.memory_required = memory_required patcher.memory_required = memory_required _auto_last["model"] = model_name def _auto_measure_begin(): """Call right before sampling: snapshot the allocator + clock.""" import time as _t try: import torch torch.cuda.reset_peak_memory_stats() return {"res": torch.cuda.memory_reserved(), "t0": _t.time()} except Exception: return {"res": None, "t0": _t.time()} def _auto_measure_end(before, patcher=None, steps=None): """Call right after sampling: cache the real pool, and DETECT the two silent failure modes by name - a system-RAM spill (peak at the card's ceiling + step time collapsed) used to present as an unexplained 5-10x slowdown with nothing in the log.""" import time as _t if not isinstance(before, dict): before = {"res": before, "t0": None} key = _auto_last["key"] if key is None: return # ---- step-time tracking (works even when CUDA stats are unavailable) sit = None if before.get("t0") and steps: sit = (_t.time() - before["t0"]) / max(1, int(steps)) base_key = "sit|" + key.rsplit("|", 1)[0] # same shape, any payload best = _auto_session.get(base_key) if best is None or sit < best: _auto_session[base_key] = sit elif best > 0 and sit > 2.5 * best: print(f"[H3AutoReserve] SLOWDOWN: {sit:.0f}s/step vs " f"{best:.0f}s/step earlier this session ({sit/best:.1f}x). " f"This is the VRAM-spill signature: the driver is paging " f"to system RAM instead of erroring. Fix: raise the " f"activation reserve, drop resolution/frames, or remove " f"reference payload (audio refs / keyframes ride every " f"step).", flush=True) if before.get("res") is None: return try: import torch peak = torch.cuda.max_memory_reserved() total = torch.cuda.get_device_properties(0).total_memory if peak >= total * 0.97: print(f"[H3AutoReserve] WARNING: peak reserved " f"{peak/2**30:.1f} GB of {total/2**30:.1f} GB - the " f"allocator hit the card's ceiling; any overflow was " f"paged to system RAM by the driver (silent, slow).", flush=True) loaded = 0 try: loaded = int(patcher.loaded_size()) if patcher is not None else 0 except Exception: try: loaded = int(getattr(patcher.model, "model_loaded_weight_memory", 0)) except Exception: loaded = 0 full = 0 try: full = int(patcher.model_size()) if patcher is not None else 0 except Exception: full = 0 # `peak` is max_memory_reserved() - VRAM. Weights that were offloaded # to the host were never in VRAM and so were never in peak. Subtract # the RESIDENT bytes, not the full model size. # # Whether the sample is usable depends on residency, in three cases: # # fully offloaded peak IS the pool, nothing to subtract. This is the # cleanest sample available - record it. On a 24 GB # card at production shapes this is the NORMAL mode, # not a fault, so discarding it stops all learning. # partial peak is bounded by what fit on the card rather than # by what the shot wanted. Measures the ceiling, not # the need. Discard. # fully resident subtract the weights and record, as always. # # An earlier version of this collapsed the first two cases and discarded # both, which wiped the cache and then left every shot reporting "first # run at this shape" - 1.5 GB reserves, 24.3 GB of driver spill and ~2x # step time on the 3090. Diagnosed there with before/after logs. pool = peak - before["res"] - loaded _frac = (loaded / float(full)) if full else 1.0 _partial = bool(full) and 0.02 < _frac < 0.98 # 2.2.5: a blanket discard on every partial load DEADLOCKS the chained # shots. Field log 2026-08-14, 5090, Q8_0 at 736x1280x243: shot 1 loads # completely and measures fine, shots 2-4 carry the context pin plus the # self-anchor audio ref, cannot fit the 20.8 GB DiT beside them, load # partially - and are then discarded. Their payload key therefore never # receives a single measurement, so it keeps the fallback reserve, so it # keeps loading partially. Learning requires a full load; a full load # requires the knowledge the rule refuses to record. Every shot after the # first ran 57 s/it against shot 1's 39. # # The discard is only actually justified when the peak was CEILING-BOUND # - i.e. the run pressed against the card and the pool got truncated. If # peak sits well below total VRAM the activations completed normally and # the pool figure is honest, even though some weights were streaming. So # gate the discard on proximity to the ceiling instead of on residency # alone, and when we do discard, still keep the observation as a FLOOR so # the estimate can only ratchet toward the truth rather than never move. _total = 0 try: import torch as _t _total = int(_t.cuda.get_device_properties(0).total_memory) except Exception: _total = 0 _ceiling_bound = bool(_total) and peak > _total * 0.94 if _partial and _ceiling_bound: _floor = _auto_cache_floor(key, pool) print(f"[H3AutoReserve] measurement capped: only " f"{loaded/2**30:.1f} of {full/2**30:.1f} GB of weights were " f"resident AND peak {peak/2**30:.1f} GB pressed the " f"{_total/2**30:.1f} GB card, so the pool was truncated. " f"Keeping {pool/2**30:.1f} GB as a floor" f"{' (raised)' if _floor else ''}; not treating it as the " f"full need.", flush=True) elif _partial: # streaming weights, but the pool itself was never squeezed if pool > 512 * 1024**2: _auto_cache_store(key, pool) _auto_session.pop(key, None) print(f"[H3AutoReserve] measured pool {pool/2**30:.1f} GB with " f"weights streaming ({loaded/2**30:.1f} of " f"{full/2**30:.1f} GB resident). Peak {peak/2**30:.1f} GB " f"stayed clear of the {_total/2**30:.1f} GB ceiling, so " f"the activation figure is sound - recording it. This is " f"the sample chained shots could never contribute before.", flush=True) elif bool(full) and _frac <= 0.02 and pool > 512 * 1024**2: # ~0% of the weights were resident: the run streamed everything, # and with the whole card to itself the pool inflates to fill # whatever reserve it was handed (allocator caches). If the # figure hugs the reserve it is an artifact OF the reserve, and # storing it locks the shape into all-streaming forever - # measured 2026-08-16 on the 3090: a cancelled 21.8 GB-reserve # run recorded a "21.7 GB pool" at a shape two healthy runs had # measured at 14-15. _pinned = _auto_session.get(key) or 0 if _pinned and pool >= _pinned * 0.85: print(f"[H3AutoReserve] discarding this shot's pool figure " f"({pool/2**30:.1f} GB): no weights were resident and " f"it hugs the {_pinned/2**30:.1f} GB reserve, so it " f"measures the reserve, not the need.", flush=True) else: _auto_cache_store(key, pool) _auto_session.pop(key, None) print(f"[H3AutoReserve] measured pool {pool/2**30:.1f} GB " f"with all weights streaming - it sits well under the " f"reserve, so the figure is real need.", flush=True) elif pool > 512 * 1024**2: # ignore no-op runs _auto_cache_store(key, pool) _auto_session.pop(key, None) # re-pin from measured print(f"[H3AutoReserve] measured pool {pool/2**30:.1f} GB for " f"this shape+payload (peak {peak/2**30:.1f} - resident " f"weights {loaded/2**30:.1f}) - next run reserves " f"{max(pool*_AUTO_MARGIN, 2*1024**3)/2**30:.1f} GB", flush=True) except Exception: pass class H3ModelLoaderAny: """One dropdown, both formats: .safetensors loads through comfy core, .gguf routes through ComfyUI-GGUF (patched for minimax_h3). Keeps the published workflow at exactly one loader node.""" @staticmethod def _list_names(folder="diffusion_models"): """Every model file the dropdown offers: core's list plus a RECURSIVE walk for .gguf, which is not in supported_pt_extensions so get_filename_list never returns it (and a flat listdir misses anything filed under diffusion_models/gguf/). Shared by INPUT_TYPES and _resolve_name - the resolver used to consult get_filename_list alone, so its "moved into a gguf/ subfolder" fallback could never find a gguf (2.6.0, found on the 24 GB box).""" import folder_paths import os try: files = list(folder_paths.get_filename_list(folder)) except Exception: files = [] gguf = [] try: dirs = folder_paths.get_folder_paths(folder) except Exception: dirs = [] for d in dirs: if not os.path.isdir(d): continue for root, _dirs, fs in os.walk(d): for f in fs: if f.lower().endswith(".gguf"): gguf.append(os.path.relpath(os.path.join(root, f), d)) return sorted(set(files) | set(gguf)) @classmethod def INPUT_TYPES(cls): names = cls._list_names("diffusion_models") return {"required": {"model_name": (names, { "tooltip": "safetensors or GGUF - loader routes automatically."})}, "optional": {"activation_reserve_gb": ("FLOAT", { "default": 0.0, "min": -1.0, "max": 128.0, "step": 0.5, "tooltip": "0 = AUTO (recommended). The pack sizes the " "activation reserve for the actual render shape, measures the " "real peak each run, and tightens itself per machine - lower " "resolutions get faster automatically. Set a number only to " "pin the reserve by hand; that number is for ONE resolution " "and the wrong number is 5-10x slower, not a little slower. " "-1 = OFF: leave ComfyUI's stock estimator alone (issue #17; " "for cards/setups where the auto-reserve mis-plans - you " "lose the leftover sweep and per-shape learning)."})}} RETURN_TYPES = ("MODEL",) FUNCTION = "load" CATEGORY = "loaders/minimax" @classmethod def VALIDATE_INPUTS(cls, model_name=None, **kwargs): # ComfyUI validates every combo value BEFORE anything runs, so a # workflow saved on a box that keeps the file at the root fails on a # box that filed it under gguf/ (or vice versa) with value_not_in_list # and nothing renders (24 GB lab F007: all three shipped workflows # unqueueable there). Accept the value here; load() resolves a moved # file by unique basename and raises a message that names the file # when it truly is not there. Same pattern RiftPromptSource uses. return True @staticmethod def _resolve_name(model_name): """F007 (24 GB lab): a saved combo value stops matching the moment the model tree is reorganised (a file moved into a gguf/ subfolder, or a workflow saved on a box that keeps it at the root). Rather than fail the whole queue with value_not_in_list, fall back to a UNIQUE basename match across the folders this loader reads, and say what was resolved. Ambiguous (two files, same basename) stays an error - guessing wrong is worse than stopping.""" import folder_paths import os want = os.path.basename(str(model_name)).lower() cands = [] for folder in ("diffusion_models", "unet", "checkpoints"): try: for f in H3ModelLoaderAny._list_names(folder): if os.path.basename(f).lower() == want: cands.append((folder, f)) except Exception: pass # exact match wins silently for folder, f in cands: if f == model_name: return model_name uniq = sorted(set(f for _, f in cands)) if len(uniq) == 1 and uniq[0] != model_name: print("[H3ModelLoader] %r is not at the saved path; resolved by " "basename to %r (the model tree moved since this workflow " "was saved)." % (model_name, uniq[0]), flush=True) return uniq[0] return model_name @staticmethod def _warn_quant_backend(model_name): """24 GB lab finding, 2026-08-17: on torch < cu130 comfy-kitchen's cuda backend is disabled and triton is off unless --enable-triton-backend, so every comfy-native quantised op (w4a8/int8/nvfp4/fp8) runs the EAGER fallback - weights dequantised to bf16 before each matmul. Measured A/B, one flag, same seed: 36.7 -> 17.6 s/it (-52%), and ~1.6x the activation pool. GGUF bypasses comfy-kitchen entirely and is unaffected. Say it once at load so nobody pays 2x for a checkpoint they chose for speed.""" n = str(model_name).lower() if n.endswith(".gguf"): return try: import comfy_kitchen as ck b = ck.list_backends() except Exception: return def live(name): info = b.get(name) if isinstance(b, dict) else None return bool(info) and info.get("available") and not info.get("disabled") if live("cuda") or live("triton"): return print("[H3ModelLoader] NOTE: only comfy-kitchen's 'eager' backend is " "live on this box (cuda needs torch cu130+; triton needs " "--enable-triton-backend). Comfy-native quantised checkpoints " "(w4a8/int8/nvfp4/fp8) dequantise to bf16 every step here - " "measured ~2x slower and ~1.6x the activation pool. Either add " "--enable-triton-backend to your launch line, or use the GGUF " "build of this model, which is unaffected.", flush=True) def load(self, model_name, activation_reserve_gb=0.0): model_name = self._resolve_name(model_name) self._warn_quant_backend(model_name) out = self._load_inner(model_name) patcher = out[0] # stash the checkpoint name so samplers can check task compatibility # (fl2va = first/last-frame hand-off, ref2va = reference rows). The # wrong pairing does not error - it silently underperforms. try: patcher.model.h3_checkpoint_name = str(model_name) except Exception: pass if activation_reserve_gb is not None and activation_reserve_gb < 0: print("[H3ModelLoader] activation reserve OFF (-1): ComfyUI's " "stock estimator is in charge. No leftover sweep, no " "per-shape learning (issue #17 escape hatch).", flush=True) elif activation_reserve_gb and activation_reserve_gb > 0: _cap = int(activation_reserve_gb * (1024 ** 3)) # Must live on the inner BaseModel, not the ModelPatcher: LoRA # stacks and guiders clone() the patcher before sampling and an # instance attribute does not survive the clone, silently # restoring comfy's estimate. Clones share this BaseModel. patcher.model.memory_required = lambda *a, _c=_cap, **k: _c patcher.memory_required = lambda *a, _c=_cap, **k: _c print(f"[H3ModelLoader] activation reserve PINNED at " f"{activation_reserve_gb:.1f} GB (manual - correct for one " f"resolution only; 0 = auto adapts to any)", flush=True) else: _install_auto_reserve(patcher, model_name) return out def _load_inner(self, model_name): import folder_paths # With VALIDATE_INPUTS accepting any name, the not-found case lands # here instead of at queue time - so say it clearly, with the folder # ComfyUI actually searched, before either loader path gets a chance # to fail obscurely. try: _found = folder_paths.get_full_path("diffusion_models", model_name) except Exception: _found = None if not _found: import os as _os _roots = [d for d in folder_paths.get_folder_paths("diffusion_models")] _hit = None for _d in _roots: if _os.path.isfile(_os.path.join(_d, model_name)): _hit = _os.path.join(_d, model_name) break if not _hit: raise RuntimeError( "[H3ModelLoader] model file not found: %r. Searched: %s. " "Pick your model file in this node's dropdown (a workflow " "saved on another machine remembers a name your models " "folder does not have)." % (model_name, ", ".join(_roots))) if model_name.lower().endswith(".gguf"): # resolve the live UnetLoaderGGUF from the global registry - # custom node packages load under mangled module names, so the # registry is the only stable handle. import nodes as core_nodes cls = core_nodes.NODE_CLASS_MAPPINGS.get("UnetLoaderGGUF") if cls is None: raise RuntimeError( "ComfyUI-GGUF not loaded - install/enable it and restart.") # ComfyUI-GGUF rejects unknown architectures before reading any # tensor, and upstream does not know minimax_h3. Import-time # patching covers the packaged install; re-assert here in case # ComfyUI-GGUF loaded after us. The relative import only exists # in the packaged install - LOOSE-FILE installs (this file # dropped straight into custom_nodes/) have no parent package, # so fall back to doing the patch inline. try: from .h3_gguf_arch import ensure_minimax_arch ensure_minimax_arch() except ImportError: import sys as _sys for _m in list(_sys.modules.values()): try: if (_m is not None and isinstance(getattr(_m, "IMG_ARCH_LIST", None), set) and hasattr(_m, "TXT_ARCH_LIST")): if "minimax_h3" not in _m.IMG_ARCH_LIST: _m.IMG_ARCH_LIST.add("minimax_h3") print("[H3ModelLoader] taught ComfyUI-GGUF " "the 'minimax_h3' architecture (in " "memory, loose-file fallback)", flush=True) break except Exception: continue return cls().load_unet(model_name) import comfy.sd path = folder_paths.get_full_path_or_raise("diffusion_models", model_name) return (comfy.sd.load_diffusion_model(path),) _UPSCALER_UTILS = None def _load_upscaler_utils(): """Load ComfyUI-MiniMaxH3_LatentUpscaler's utils.py by path (cached). Two-pass upscale reuses that pack's NestedTensor upscale + CONST re-noise math rather than duplicating it. Works whether this file is installed loose in custom_nodes or inside ComfyUI-H3-Multishot/. """ global _UPSCALER_UTILS if _UPSCALER_UTILS is not None: return _UPSCALER_UTILS import importlib.util import os here = os.path.dirname(os.path.abspath(__file__)) candidates = [ os.path.join(here, "ComfyUI-MiniMaxH3_LatentUpscaler", "utils.py"), os.path.join(os.path.dirname(here), "ComfyUI-MiniMaxH3_LatentUpscaler", "utils.py"), ] try: import folder_paths for d in folder_paths.get_folder_paths("custom_nodes"): candidates.append( os.path.join(d, "ComfyUI-MiniMaxH3_LatentUpscaler", "utils.py")) except Exception: pass for path in candidates: if os.path.isfile(path): spec = importlib.util.spec_from_file_location( "h3_latent_upscaler_utils", path) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) _UPSCALER_UTILS = mod return mod raise RuntimeError( "two_pass_upscale needs the ComfyUI-MiniMaxH3_LatentUpscaler pack " "(github.com/Tr1dae/ComfyUI-MiniMaxH3_LatentUpscaler) installed in " "custom_nodes - it provides the NestedTensor upscale/re-noise math.") def _upscale_av_exact(tr, latent_dict, target_h, target_w): """Spatially upscale the VIDEO member of an AV latent to EXACT latent dims. The pack's own upscaler works from a single scale_by, which can round to a grid one cell off the requested resolution; sampling to a fixed widget resolution needs exact dims so every shot decodes to width x height. """ import torch.nn.functional as F members, was_nested = tr.extract_tensor(latent_dict["samples"]) v = members[0] orig = tuple(v.shape) x = v if len(orig) > 4: # [B,C,T,H,W] -> [B*T,C,H,W] x = x.reshape(orig[0], orig[1], -1, orig[-2], orig[-1]) x = x.movedim(2, 1).reshape(-1, orig[1], orig[-2], orig[-1]) x = F.interpolate(x, size=(target_h, target_w), mode="bilinear", align_corners=False) if len(orig) > 4: x = x.reshape(orig[0], -1, orig[1], target_h, target_w).movedim(2, 1) out = latent_dict.copy() out["samples"] = tr.wrap_tensor([x, *members[1:]], was_nested=was_nested) return out # --------------------------------------------------------------------------- # Chain gain control (seam sharpening) # # MEASURED 2026-08-08 on 6 real multishot renders, both rigs, no LoRA: every # content-continuous seam steps UP in texture energy by +25..47% (5-7x the # non-seam control), and because each shot tail becomes the next shot keyframe # anchor, the level COMPOUNDS - a 6-shot chain measured 3.3x sharper by its # last shot than its first. The loop: anchor at level L -> the model generates # its continuation at ~1.25-1.47 x L -> that output tail is the next anchor. # Gain > 1 in an autoregressive loop ratchets. # # These helpers measure texture energy (variance of a 3x3 Laplacian on luma) # and apply a separable gaussian, so the chain can be level-controlled. # --------------------------------------------------------------------------- def _cg_lap_var(img): """CONTRAST-NORMALISED texture energy of an IMAGE batch [B,H,W,C] in 0..1. var(laplacian) / var(luma). The raw laplacian scales with local contrast, so on its own it cannot tell "the shot got brighter" from "the shot got sharper" - and a leveller driven by it chases exposure instead of detail (measured: it held the answering-machine scene flat, whose contrast was steady, and missed a 2.7x drift in the dim coma scene, whose contrast swung 0.13-0.19). Dividing by the frame's own variance measures detail SCALE, which is what must stay constant across a chain. Letterbox bars are excluded - pure black borders would drag both terms. """ import torch import torch.nn.functional as F x = img if img.ndim == 4 else img.unsqueeze(0) g = (x[..., 0] * 0.299 + x[..., 1] * 0.587 + x[..., 2] * 0.114).unsqueeze(1) if g.shape[-1] > 8 and g.shape[-2] > 8: mx_r = g.amax(dim=(0, 1, 3)) > 0.02 # rows with any signal mx_c = g.amax(dim=(0, 1, 2)) > 0.02 # cols with any signal if bool(mx_r.any()) and bool(mx_c.any()): g = g[:, :, mx_r][:, :, :, mx_c] k = torch.tensor([[0., 1., 0.], [1., -4., 1.], [0., 1., 0.]], dtype=g.dtype, device=g.device).view(1, 1, 3, 3) lap = float(F.conv2d(g, k, padding=1).var()) con = float(g.var()) return lap / max(con, 1e-9) def _cg_gauss(img, sigma): """Separable gaussian blur on an IMAGE batch [B,H,W,C].""" import torch import torch.nn.functional as F if sigma <= 0: return img r = max(1, int(round(sigma * 3))) xs = torch.arange(-r, r + 1, dtype=img.dtype, device=img.device) k = torch.exp(-(xs ** 2) / (2 * sigma * sigma)) k = k / k.sum() v = img.permute(0, 3, 1, 2) # [B,C,H,W] c = v.shape[1] kh = k.view(1, 1, 1, -1).expand(c, 1, 1, k.numel()) v = F.conv2d(F.pad(v, (r, r, 0, 0), mode="reflect"), kh, groups=c) kv = k.view(1, 1, -1, 1).expand(c, 1, k.numel(), 1) v = F.conv2d(F.pad(v, (0, 0, r, r), mode="reflect"), kv, groups=c) return v.permute(0, 2, 3, 1).clamp(0, 1) def _cg_sigma_for(img, target_lap, max_sigma=1.6): """Smallest gaussian sigma bringing img texture energy down to target.""" if target_lap <= 0: return 0.0 # fine steps: the lap response to sigma is steep below ~0.7, so a coarse # grid overshoots badly (measured on synthetic texture: 0.3 -> 0.97x but # 0.6 -> 0.31x). 0.05 keeps the landing within a few percent. best_s, best_d = 0.0, abs(_cg_lap_var(img) - target_lap) s = 0.05 while s <= max_sigma + 1e-9: d = abs(_cg_lap_var(_cg_gauss(img, s)) - target_lap) if d < best_d: best_d, best_s = d, s s += 0.05 return best_s def _cg_flatten(imgs, target, block=8, max_sigma=1.6): """Level a shot to a constant texture energy: blur only, per block. Sigma is searched on one representative frame per block (cheap) and then smoothed across blocks so the correction cannot pump frame to frame. Frames already at or below target are left untouched - this never sharpens, so it can only remove drift, never invent detail. """ import torch n = imgs.shape[0] if n == 0 or target <= 0: return imgs, 0.0 idx = list(range(0, n, block)) sig = [] for i in idx: f = imgs[i:i + 1] sig.append(_cg_sigma_for(f, target, max_sigma) if _cg_lap_var(f) > target * 1.02 else 0.0) # smooth (moving average of 3) so sigma cannot jump between blocks sm = [] for j in range(len(sig)): lo, hi = max(0, j - 1), min(len(sig), j + 2) sm.append(sum(sig[lo:hi]) / (hi - lo)) if max(sm) <= 0.0: return imgs, 0.0 out = imgs.clone() for j, i in enumerate(idx): s = sm[j] if s > 0.02: out[i:i + block] = _cg_gauss(imgs[i:i + block], s) return out, (sum(sm) / len(sm)) def _sampler_names(): """From core, so the list cannot rot out of step with ComfyUI.""" try: import comfy.samplers return list(comfy.samplers.KSampler.SAMPLERS) except Exception: return ["res_multistep", "euler", "dpmpp_2m"] def _scheduler_names(): try: import comfy.samplers return list(comfy.samplers.KSampler.SCHEDULERS) except Exception: return ["simple", "normal", "beta"] def _mmproj_postprocess(gg_loader, vsd, label): """Everything gguf_mmproj_loader does AFTER gguf_sd_loader. The explicit-file path skipped all of it, so the vision tower loaded under raw llama.cpp names (v.blk.*, mm.*) that nothing downstream consumes - user-reported as a matmul shape error mid-render, cured by switching back to (auto). Same file, different loader. Uses upstream's own map and helpers rather than reimplementing the rename, so if their key map changes this follows it. If their internals move, fail loudly here with an instruction rather than silently returning half a vision tower. """ import torch try: sd_map_replace = gg_loader.sd_map_replace CLIP_VISION_SD_MAP = gg_loader.CLIP_VISION_SD_MAP dequantize_tensor = gg_loader.dequantize_tensor is_quantized = gg_loader.is_quantized except AttributeError as e: raise RuntimeError( f"This ComfyUI-GGUF build does not expose the vision key map " f"({e}), so an explicitly chosen mmproj cannot be renamed to the " f"layout the encoder expects. Set mmproj_name back to (auto) and " f"keep the mmproj beside the encoder with a matching name.") # 1. 4D patch_embd pair -> 5D if "v.patch_embd.weight.1" in vsd: w1 = dequantize_tensor(vsd.pop("v.patch_embd.weight"), dtype=torch.float32) w2 = dequantize_tensor(vsd.pop("v.patch_embd.weight.1"), dtype=torch.float32) vsd["v.patch_embd.weight"] = torch.stack([w1, w2], dim=2) # 2. the rename that makes the tower addressable at all vsd = sd_map_replace(vsd, CLIP_VISION_SD_MAP) # 3. split q/k/v -> fused qkv if "visual.blocks.0.attn_q.weight" in vsd: attns = {} for k, v in vsd.items(): if any(x in k for x in ("attn_q", "attn_k", "attn_v")): k_attn, k_name = k.rsplit(".attn_", 1) k_attn += ".attn.qkv." + k_name.split(".")[-1] attns.setdefault(k_attn, {})[k_name] = dequantize_tensor( v, dtype=(torch.bfloat16 if is_quantized(v) else torch.float16)) for k, v in attns.items(): sfx = k.split(".")[-1] vsd[k] = torch.cat([v[f"q.{sfx}"], v[f"k.{sfx}"], v[f"v.{sfx}"]], dim=0) if not any(k.startswith("visual.") for k in vsd): raise RuntimeError( f"The mmproj '{label}' loaded but produced no visual.* tensors, so " f"it is not a vision sidecar for this encoder. Pick the -mmproj " f"file that belongs to the encoder you selected, or set " f"mmproj_name to (auto).") return vsd class H3ClipLoaderAny: """One dropdown for text encoders, both formats: .safetensors through comfy core CLIPLoader, .gguf through ComfyUI-GGUF's CLIPLoaderGGUF (which auto-pairs a matching -mmproj sidecar for vision).""" @classmethod def INPUT_TYPES(cls): import os import folder_paths files = set(folder_paths.get_filename_list("text_encoders")) for d in folder_paths.get_folder_paths("text_encoders"): if not os.path.isdir(d): continue # RECURSIVE, same reason as the model loader above. for root, _dirs, fs in os.walk(d): for f in fs: if f.lower().endswith(".gguf") and "mmproj" not in f.lower(): files.add(os.path.relpath(os.path.join(root, f), d)) mm = ["(auto)"] for d in folder_paths.get_folder_paths("text_encoders"): if not os.path.isdir(d): continue for root, _dirs, fs in os.walk(d): for f in fs: if f.lower().endswith(".gguf") and "mmproj" in f.lower(): mm.append(os.path.relpath(os.path.join(root, f), d)) import nodes as core_nodes types = core_nodes.CLIPLoader.INPUT_TYPES()["required"]["type"] return {"required": { "clip_name": (sorted(files), { "tooltip": "safetensors or GGUF - routed automatically. GGUF " "encoders auto-pair their -mmproj vision sidecar."}), "type": types, }, "optional": { # NEW WIDGETS APPEND LAST - inserting above shifts every saved # workflow's values by one slot. "mmproj_name": (mm, { "default": "(auto)", "tooltip": "Vision sidecar for a GGUF encoder. '(auto)' uses " "ComfyUI-GGUF's pairing, which matches on FILENAME " "inside the encoder's own folder - rename either " "file, or split them across folders, and the match " "fails. If auto finds nothing and exactly one " "mmproj sits beside the encoder, that one is used " "anyway. Pick a file here to override entirely; " "then names and folders do not matter."}), }} RETURN_TYPES = ("CLIP",) FUNCTION = "load" CATEGORY = "loaders/minimax" # llama.cpp/qwen2vl-era names -> the H3 encoder's exact visual.* layout # (established 2026-08-04 against the official int8 file). Ordered, and # chosen so no rule can re-hit another rule's output. _VISION_FIXES = [ ("visual.merger.ln_q.", "visual.merger.norm."), ("attn_qkv.", "attn.qkv."), ("mlp.up_proj.", "mlp.linear_fc1."), ("mlp.down_proj.", "mlp.linear_fc2."), (".fc1.", ".linear_fc1."), (".fc2.", ".linear_fc2."), ("v.position_embd.weight", "visual.pos_embed.weight"), ] def load(self, clip_name, type, mmproj_name="(auto)"): import os import re import sys import nodes as core_nodes if not clip_name.lower().endswith(".gguf"): return core_nodes.CLIPLoader().load_clip(clip_name, type=type) gg_cls = core_nodes.NODE_CLASS_MAPPINGS.get("CLIPLoaderGGUF") if gg_cls is None: raise RuntimeError( "ComfyUI-GGUF not loaded - install/enable it and restart.") gg = sys.modules[gg_cls.__module__] # gguf_mmproj_loader lives in their loader module, which nodes.py # does not re-export - resolve it from where gguf_clip_loader is # actually defined. gg_loader = sys.modules[gg.gguf_clip_loader.__module__] import folder_paths import comfy.sd import comfy.model_management clip_path = folder_paths.get_full_path("clip", clip_name) # --- text side: their mapper, then truncate to the official H3 # shape (Qwen3-VL-32B cut to 50 layers; no final norm, no lm_head). sd = gg.gguf_clip_loader(clip_path) drop = re.compile(r"model\.layers\.(5[0-9]|6[0-9])\.") sd = {k: v for k, v in sd.items() if not drop.match(k) and k not in ("model.norm.weight", "lm_head.weight")} # --- vision side: their sidecar loader, then correct the names to # H3's layout (their map is qwen2vl-era: wrong merger keys, missing # deepstack and qkv rules). # Upstream pairs the sidecar by FILENAME inside the encoder's own # folder (gguf_mmproj_loader: strip quant suffix, substring match), # so a rename on either file - or splitting them across folders - # breaks the pair. Upstream then logs an error and returns {}, which # renders as "the model ignores my reference image". Three ways out, # in order, and a hard failure rather than a silent one. vsd = None if mmproj_name and mmproj_name != "(auto)": mm_path = folder_paths.get_full_path("text_encoders", mmproj_name) if not mm_path: raise RuntimeError( f"mmproj_name '{mmproj_name}' is not in the " f"text_encoders folder any more.") vsd, _ = gg_loader.gguf_sd_loader(mm_path, is_text_model=True) vsd = _mmproj_postprocess(gg_loader, vsd, mmproj_name) print(f"[H3ClipLoader] vision sidecar (explicit): {mmproj_name}", flush=True) else: vsd = gg_loader.gguf_mmproj_loader(clip_path) if not vsd: # upstream's name match failed; if exactly one mmproj sits # beside the encoder the intent is unambiguous, so use it _dir = os.path.dirname(clip_path) cands = [f for f in os.listdir(_dir) if f.lower().endswith(".gguf") and "mmproj" in f.lower()] if len(cands) == 1: vsd, _ = gg_loader.gguf_sd_loader( os.path.join(_dir, cands[0]), is_text_model=True) print(f"[H3ClipLoader] filename pairing failed, but " f"'{cands[0]}' is the only mmproj beside the " f"encoder - using it. Set mmproj_name to silence " f"this.", flush=True) if not vsd: raise RuntimeError( f"No vision sidecar (-mmproj) resolved for '{clip_name}'. " f"The H3 encoder NEEDS its vision tower for image " f"references and shot chaining. Either keep the mmproj file " f"beside the encoder with a matching name, or just pick it " f"explicitly in this node's mmproj_name widget - with that " f"set, names and folders do not matter.") # merger mlp indices -> linear_fc1/2 by ascending index idxs = sorted({m.group(1) for k in vsd for m in [re.match(r"visual\.merger\.mlp\.(\d+)\.", k)] if m}) # deepstack mergers: llama.cpp indexes them by the vision layer they # tap (8/16/24), the H3 encoder by list position (0/1/2) - remap # ascending, and sort NUMERICALLY (lexically 16 < 8). ds = sorted({int(m.group(1)) for k in vsd for m in [re.match(r"v\.deepstack\.(\d+)\.", k)] if m}) fixed = {} for k, v in vsd.items(): for i, name in zip(idxs, ("linear_fc1", "linear_fc2")): k = k.replace(f"visual.merger.mlp.{i}.", f"visual.merger.{name}.") for pos, layer in enumerate(ds): k = k.replace(f"v.deepstack.{layer}.", f"visual.deepstack_merger_list.{pos}.") for a, b in self._VISION_FIXES: k = k.replace(a, b) fixed[k] = v sd.update(fixed) clip = comfy.sd.load_text_encoder_state_dicts( clip_type=getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION), state_dicts=[sd], model_options={ "custom_operations": gg.GGMLOps, "initial_device": comfy.model_management.text_encoder_offload_device(), }, embedding_directory=folder_paths.get_folder_paths("embeddings"), ) clip.patcher = gg.GGUFModelPatcher.clone(clip.patcher) return (clip,) class H3AudioTrimStart: """Trim N seconds off the FRONT of an audio clip. Exists so the multishot master can drop each chained shot's duplicated first frame (1/24s) from video AND audio together, keeping lip sync exact across seams.""" @classmethod def INPUT_TYPES(cls): return {"required": { "audio": ("AUDIO",), "seconds": ("FLOAT", {"default": 0.04167, "min": 0.0, "max": 10.0, "step": 0.00001}), }} RETURN_TYPES = ("AUDIO",) FUNCTION = "trim" CATEGORY = "audio" def trim(self, audio, seconds): sr = audio["sample_rate"] wav = audio["waveform"] n = int(round(seconds * sr)) return ({"sample_rate": sr, "waveform": wav[..., n:]},) class H3MultishotSampler: """The whole multishot pipeline in one node: parse script, loop shots, chain each shot's last frame into the next shot's first_frame, seam-trim, and return master frames + master audio. shot_count is REAL here: N shots means N sampled shots, no wasted execution. JoyEcho architecture applied to H3: multishot complexity lives inside the node so the canvas stays legible.""" @classmethod def INPUT_TYPES(cls): return {"required": { "model": ("MODEL",), "clip": ("CLIP",), "video_vae": ("VAE",), "audio_vae": ("VAE",), "script": ("STRING", { "multiline": True, "dynamicPrompts": False, "default": "Shot 1 prompt goes here.\n---\n" "Shot 2 prompt goes here.\n---\n" "Shot 3 prompt goes here.", "tooltip": "One prompt per shot, separated by --- on its own " "line. JSON {\"prompts\": [...]} also accepted."}), "shot_count": ("INT", { "default": 0, "min": 0, "max": 8, "tooltip": "The TOTAL number of shots - not shots per prompt. " "Leave it at 0 and the script decides: one shot per " "--- block, which is what you want for a written scene. " "1-8 forces the total instead: extra blocks are dropped, " "and if the script is short the last block repeats as a " "continuation. Every shot renders."}), "width": ("INT", {"default": 768, "min": 32, "max": 4096, "step": 32}), "height": ("INT", {"default": 1344, "min": 32, "max": 4096, "step": 32}), "frames_per_shot": ("INT", { "default": 243, "min": 5, "max": 481, "step": 17, "tooltip": "Frames at 24fps on H3's 17k+5 grid (243 = ~10.1s;" " 362 = trained max ~15.1s; beyond is untested)."}), "seed": ("INT", {"default": 0, "min": 0, "max": 0xffffffffffffffff, "control_after_generate": True}), "steps": ("INT", {"default": 20, "min": 1, "max": 50}), "seed_per_shot": ("BOOLEAN", { "default": True, "label_on": "vary per shot", "label_off": "same seed every shot", "tooltip": "Leave ON. Measured: varying the seed per shot holds the " "face across the chain; using one seed for every shot " "made BOTH the face and the voice drift. Identity " "lives in the conditioning, not the seed."}), }, "optional": { "start_image": ("IMAGE", { "tooltip": "Optional first frame (I2V). Shot 1 starts from this " "image; later shots continue chaining from the " "previous shot's last frame as usual. Leave " "unconnected for pure text-to-video."}), "reference_images": ("IMAGE", { "tooltip": "Optional SUBJECT/CHARACTER reference images (batch " "= multiple refs, e.g. via Batch Images), carried " "into EVERY shot as , , ... " "- distinct from start_image (which only seeds the " "I2V chain frame). Bind them in each shot's prompt: " "'She looks like the woman in .' " "Verified on the ref2va checkpoint."}), "voice_ref": ("AUDIO", { "tooltip": "Optional VOICE ANCHOR carried into EVERY shot as a " "reference audio (