Upload Salia_Auto_Align_Overlay.py
Browse files- Salia_Auto_Align_Overlay.py +103 -9
Salia_Auto_Align_Overlay.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
"""
|
| 2 |
-
Salia Auto Align Overlay -- continuous registration, v2.
|
| 3 |
=======================================================
|
| 4 |
Install this single file in ComfyUI/custom_nodes/, replacing the earlier
|
| 5 |
Salia_Auto_Align_Overlay.py. Restart ComfyUI and add a NEW copy of the node
|
|
@@ -33,6 +33,9 @@ Algorithm
|
|
| 33 |
1. Locate the original crop with full-resolution masked template matching
|
| 34 |
(weighted color SSD accelerated by float64 FFT correlation). Directly
|
| 35 |
verify near-best matches; report ambiguous or nonexact locations.
|
|
|
|
|
|
|
|
|
|
| 36 |
2. Jointly optimize center X/Y and independent positive width/height using
|
| 37 |
scipy.optimize.least_squares(method='trf') with a weighted soft-L1 loss.
|
| 38 |
This is a continuous four-parameter registration, with no rotation/shear
|
|
@@ -290,9 +293,11 @@ def resize_rgba_ewa(source: np.ndarray, width: int, height: int,
|
|
| 290 |
import logging
|
| 291 |
import time
|
| 292 |
from scipy.signal import correlate
|
| 293 |
-
__version__ = "2.
|
| 294 |
_LOG = logging.getLogger(__name__)
|
| 295 |
_MAX_DIMENSION = 32768
|
|
|
|
|
|
|
| 296 |
|
| 297 |
def _rgba_array(image, name="image"):
|
| 298 |
"""Private normalized float32 copy; hidden RGB is retained but never scored."""
|
|
@@ -365,7 +370,7 @@ def _overlap(background, template, x, y):
|
|
| 365 |
return (slice(y0, y1), slice(x0, x1),
|
| 366 |
slice(y0 - y, y1 - y), slice(x0 - x, x1 - x))
|
| 367 |
|
| 368 |
-
def _error_at(background, template, x, y):
|
| 369 |
"""Full template normalization: clipping outside the canvas cannot win."""
|
| 370 |
weight = _weights(template)
|
| 371 |
total = float(weight.sum())
|
|
@@ -378,10 +383,15 @@ def _error_at(background, template, x, y):
|
|
| 378 |
inside_weight = weight[ty, tx]
|
| 379 |
# Off-canvas visible pixels are maximum mismatch, not discarded samples.
|
| 380 |
outside_error = max(0.0, total - float(inside_weight.sum()))
|
| 381 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 382 |
return float(np.clip((outside_error + inside_error) / total, 0.0, 1.0))
|
| 383 |
|
| 384 |
-
def _locate_unedited(background, template, interrupt=lambda: None):
|
| 385 |
"""Evaluate every fully-contained placement using weighted RGB SSD.
|
| 386 |
|
| 387 |
FFT correlation is an algebraic acceleration of the SAME per-pixel
|
|
@@ -396,7 +406,12 @@ def _locate_unedited(background, template, interrupt=lambda: None):
|
|
| 396 |
total = float(weight.sum())
|
| 397 |
if total <= 0.0:
|
| 398 |
raise ValueError("Unedited_Rect is fully transparent; its position cannot be located.")
|
| 399 |
-
visible = (background[..., 3] > _ALPHA_EPSILON).astype(np.float64)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 400 |
rgb = background[..., :3].astype(np.float64)
|
| 401 |
template_rgb = template[..., :3].astype(np.float64)
|
| 402 |
|
|
@@ -435,7 +450,7 @@ def _locate_unedited(background, template, interrupt=lambda: None):
|
|
| 435 |
for flat in chain(candidates, (int(k) for k in near_best if k != first and k != fft_best)):
|
| 436 |
interrupt()
|
| 437 |
y, x = divmod(int(flat), errors.shape[1])
|
| 438 |
-
error = _error_at(background, template, x, y)
|
| 439 |
if error < best_error:
|
| 440 |
best_error, best_x, best_y = error, x, y
|
| 441 |
if best_error == 0.0:
|
|
@@ -489,6 +504,10 @@ def _tensor_image(image, name, mask=None):
|
|
| 489 |
raise ValueError("%s_mask contains NaN or infinity." % name)
|
| 490 |
# Comfy Load Image MASK is 1-alpha; supplying it REPLACES image alpha.
|
| 491 |
result[..., 3] = 1.0 - np.clip(value, 0.0, 1.0)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 492 |
return result
|
| 493 |
|
| 494 |
# ---------------------------------------------------------------------------
|
|
@@ -748,6 +767,14 @@ def _validate_registration_options(target_width, target_height, max_evaluations,
|
|
| 748 |
raise ValueError("%s must be a finite number in [%g,%g]." % (name, lo, hi))
|
| 749 |
|
| 750 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 751 |
def align_arrays(background, overlay, unedited_rect, target_width=0, target_height=0,
|
| 752 |
max_evaluations=100, pyramid_levels=3, robust_threshold=0.05,
|
| 753 |
scale_limit=2.0, interrupt=lambda: None, progress=lambda value: None):
|
|
@@ -760,13 +787,76 @@ def align_arrays(background, overlay, unedited_rect, target_width=0, target_heig
|
|
| 760 |
started = time.monotonic()
|
| 761 |
_validate_registration_options(target_width, target_height, max_evaluations,
|
| 762 |
pyramid_levels, robust_threshold, scale_limit)
|
|
|
|
| 763 |
background = _rgba_array(background, "background")
|
| 764 |
overlay = _rgba_array(overlay, "overlay")
|
| 765 |
unedited_rect = _rgba_array(unedited_rect, "Unedited_Rect")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 766 |
if float(_weights(overlay).sum()) <= 0.0:
|
| 767 |
raise ValueError("overlay is fully transparent; there is nothing to align or paste.")
|
| 768 |
interrupt()
|
| 769 |
crop_x, crop_y, crop_error, ties = _locate_unedited(background, unedited_rect, interrupt)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 770 |
crop_height, crop_width = unedited_rect.shape[:2]
|
| 771 |
center_x, center_y = crop_x + crop_width / 2.0, crop_y + crop_height / 2.0
|
| 772 |
width0, height0 = float(target_width or crop_width), float(target_height or crop_height)
|
|
@@ -796,7 +886,7 @@ def align_arrays(background, overlay, unedited_rect, target_width=0, target_heig
|
|
| 796 |
lower[2:] = np.minimum(lower[2:], -1.0e-12)
|
| 797 |
upper[2:] = np.maximum(upper[2:], 1.0e-12)
|
| 798 |
parameters = np.zeros(4, dtype=np.float64)
|
| 799 |
-
trace, notices = [],
|
| 800 |
final_problem = None
|
| 801 |
last_result = None
|
| 802 |
total_calls = 0
|
|
@@ -871,6 +961,8 @@ def align_arrays(background, overlay, unedited_rect, target_width=0, target_heig
|
|
| 871 |
% (state, "solver converged numerically" if converged else "inspect solver notes"),
|
| 872 |
"Crop: top_left=(%d,%d), size=%dx%d, similarity=%.8f/200"
|
| 873 |
% (crop_x, crop_y, crop_width, crop_height, 200.0 * (1.0 - crop_error)),
|
|
|
|
|
|
|
| 874 |
"Overlay: top_left=(%.8f,%.8f), size=(%.8f,%.8f), center=(%.8f,%.8f)"
|
| 875 |
% (x, y, width, height, center_x, center_y),
|
| 876 |
"Rendered similarity=%.8f/200; registration similarity=%.8f/200; robust cost %.10g -> %.10g"
|
|
@@ -890,7 +982,9 @@ def align_arrays(background, overlay, unedited_rect, target_width=0, target_heig
|
|
| 890 |
"center_x": float(center_x), "center_y": float(center_y), "similarity": similarity,
|
| 891 |
"registration_similarity": registration_similarity, "robust_cost": fit_cost,
|
| 892 |
"initial_robust_cost": base_cost, "converged": converged, "accepted": accepted,
|
| 893 |
-
"trace": trace, "parameters": parameters.copy(), "crop_x": crop_x, "crop_y": crop_y
|
|
|
|
|
|
|
| 894 |
|
| 895 |
|
| 896 |
class Salia_Auto_Align_Overlay:
|
|
|
|
| 1 |
"""
|
| 2 |
+
Salia Auto Align Overlay -- continuous registration, v2.1
|
| 3 |
=======================================================
|
| 4 |
Install this single file in ComfyUI/custom_nodes/, replacing the earlier
|
| 5 |
Salia_Auto_Align_Overlay.py. Restart ComfyUI and add a NEW copy of the node
|
|
|
|
| 33 |
1. Locate the original crop with full-resolution masked template matching
|
| 34 |
(weighted color SSD accelerated by float64 FFT correlation). Directly
|
| 35 |
verify near-best matches; report ambiguous or nonexact locations.
|
| 36 |
+
Original crop location must score at least 199.8/200 or no overlay is
|
| 37 |
+
pasted. If an opaque/RGB crop has lost alpha, a verified near-exact RGB
|
| 38 |
+
copy match can recover its location without punishing transparent padding.
|
| 39 |
2. Jointly optimize center X/Y and independent positive width/height using
|
| 40 |
scipy.optimize.least_squares(method='trf') with a weighted soft-L1 loss.
|
| 41 |
This is a continuous four-parameter registration, with no rotation/shear
|
|
|
|
| 293 |
import logging
|
| 294 |
import time
|
| 295 |
from scipy.signal import correlate
|
| 296 |
+
__version__ = "2.1.0"
|
| 297 |
_LOG = logging.getLogger(__name__)
|
| 298 |
_MAX_DIMENSION = 32768
|
| 299 |
+
_MIN_CROP_SIMILARITY = 199.8
|
| 300 |
+
_RGB_COPY_ERROR_TOLERANCE = 1.0e-6
|
| 301 |
|
| 302 |
def _rgba_array(image, name="image"):
|
| 303 |
"""Private normalized float32 copy; hidden RGB is retained but never scored."""
|
|
|
|
| 370 |
return (slice(y0, y1), slice(x0, x1),
|
| 371 |
slice(y0 - y, y1 - y), slice(x0 - x, x1 - x))
|
| 372 |
|
| 373 |
+
def _error_at(background, template, x, y, respect_background_alpha=True):
|
| 374 |
"""Full template normalization: clipping outside the canvas cannot win."""
|
| 375 |
weight = _weights(template)
|
| 376 |
total = float(weight.sum())
|
|
|
|
| 383 |
inside_weight = weight[ty, tx]
|
| 384 |
# Off-canvas visible pixels are maximum mismatch, not discarded samples.
|
| 385 |
outside_error = max(0.0, total - float(inside_weight.sum()))
|
| 386 |
+
if respect_background_alpha:
|
| 387 |
+
pixel_error = _color_error(background[by, bx], template[ty, tx])
|
| 388 |
+
else:
|
| 389 |
+
delta = background[by, bx, :3].astype(np.float64) - template[ty, tx, :3]
|
| 390 |
+
pixel_error = np.mean(delta * delta, axis=-1)
|
| 391 |
+
inside_error = np.sum(inside_weight * pixel_error)
|
| 392 |
return float(np.clip((outside_error + inside_error) / total, 0.0, 1.0))
|
| 393 |
|
| 394 |
+
def _locate_unedited(background, template, interrupt=lambda: None, respect_background_alpha=True):
|
| 395 |
"""Evaluate every fully-contained placement using weighted RGB SSD.
|
| 396 |
|
| 397 |
FFT correlation is an algebraic acceleration of the SAME per-pixel
|
|
|
|
| 406 |
total = float(weight.sum())
|
| 407 |
if total <= 0.0:
|
| 408 |
raise ValueError("Unedited_Rect is fully transparent; its position cannot be located.")
|
| 409 |
+
visible = ((background[..., 3] > _ALPHA_EPSILON).astype(np.float64)
|
| 410 |
+
if respect_background_alpha else np.ones((bh, bw), dtype=np.float64))
|
| 411 |
+
if not np.any(visible):
|
| 412 |
+
# Every placement has the same maximum appearance error. Avoid
|
| 413 |
+
# directly rescoring every all-transparent location in a large image.
|
| 414 |
+
return 0, 0, 1.0, (bh - th + 1) * (bw - tw + 1)
|
| 415 |
rgb = background[..., :3].astype(np.float64)
|
| 416 |
template_rgb = template[..., :3].astype(np.float64)
|
| 417 |
|
|
|
|
| 450 |
for flat in chain(candidates, (int(k) for k in near_best if k != first and k != fft_best)):
|
| 451 |
interrupt()
|
| 452 |
y, x = divmod(int(flat), errors.shape[1])
|
| 453 |
+
error = _error_at(background, template, x, y, respect_background_alpha)
|
| 454 |
if error < best_error:
|
| 455 |
best_error, best_x, best_y = error, x, y
|
| 456 |
if best_error == 0.0:
|
|
|
|
| 504 |
raise ValueError("%s_mask contains NaN or infinity." % name)
|
| 505 |
# Comfy Load Image MASK is 1-alpha; supplying it REPLACES image alpha.
|
| 506 |
result[..., 3] = 1.0 - np.clip(value, 0.0, 1.0)
|
| 507 |
+
# Preserve the fact that RGB had NO alpha. Promoting it here used to erase
|
| 508 |
+
# that information before crop-location diagnostics and recovery.
|
| 509 |
+
if mask is None and int(image.shape[-1]) == 3:
|
| 510 |
+
return np.ascontiguousarray(result[..., :3])
|
| 511 |
return result
|
| 512 |
|
| 513 |
# ---------------------------------------------------------------------------
|
|
|
|
| 767 |
raise ValueError("%s must be a finite number in [%g,%g]." % (name, lo, hi))
|
| 768 |
|
| 769 |
|
| 770 |
+
def _input_alpha_description(name, rgba, input_channels):
|
| 771 |
+
alpha = rgba[..., 3]
|
| 772 |
+
transparent = float(np.mean(alpha <= _ALPHA_EPSILON) * 100.0)
|
| 773 |
+
opaque = float(np.mean(alpha >= 1.0 - _ALPHA_EPSILON) * 100.0)
|
| 774 |
+
return ("%s=%dx%d, input_channels=%d, alpha_zero=%.2f%%, alpha_opaque=%.2f%%"
|
| 775 |
+
% (name, rgba.shape[1], rgba.shape[0], input_channels, transparent, opaque))
|
| 776 |
+
|
| 777 |
+
|
| 778 |
def align_arrays(background, overlay, unedited_rect, target_width=0, target_height=0,
|
| 779 |
max_evaluations=100, pyramid_levels=3, robust_threshold=0.05,
|
| 780 |
scale_limit=2.0, interrupt=lambda: None, progress=lambda value: None):
|
|
|
|
| 787 |
started = time.monotonic()
|
| 788 |
_validate_registration_options(target_width, target_height, max_evaluations,
|
| 789 |
pyramid_levels, robust_threshold, scale_limit)
|
| 790 |
+
raw_inputs = (background, overlay, unedited_rect)
|
| 791 |
background = _rgba_array(background, "background")
|
| 792 |
overlay = _rgba_array(overlay, "overlay")
|
| 793 |
unedited_rect = _rgba_array(unedited_rect, "Unedited_Rect")
|
| 794 |
+
descriptions = [
|
| 795 |
+
_input_alpha_description(name, rgba, int(np.asarray(raw).shape[-1]))
|
| 796 |
+
for name, raw, rgba in zip(("background", "overlay", "Unedited_Rect"),
|
| 797 |
+
raw_inputs, (background, overlay, unedited_rect))
|
| 798 |
+
]
|
| 799 |
+
input_diagnostics = "Inputs:\n" + "\n".join(descriptions)
|
| 800 |
if float(_weights(overlay).sum()) <= 0.0:
|
| 801 |
raise ValueError("overlay is fully transparent; there is nothing to align or paste.")
|
| 802 |
interrupt()
|
| 803 |
crop_x, crop_y, crop_error, ties = _locate_unedited(background, unedited_rect, interrupt)
|
| 804 |
+
crop_match_mode = "alpha-aware crop appearance"
|
| 805 |
+
crop_notes = []
|
| 806 |
+
# A Load Image IMAGE output is RGB. If its MASK is missing while the
|
| 807 |
+
# background retains alpha, treating that RGB crop as genuinely opaque
|
| 808 |
+
# turns matching transparent padding into a maximum-error penalty.
|
| 809 |
+
# Recover ONLY when the actual RGB values provide a near-exact copy
|
| 810 |
+
# match. Do not cherry-pick arbitrary low-scoring fallback locations.
|
| 811 |
+
if (crop_error > 1.0e-8
|
| 812 |
+
and np.all(unedited_rect[..., 3] >= 1.0 - _ALPHA_EPSILON)
|
| 813 |
+
and np.any(background[..., 3] <= _ALPHA_EPSILON)):
|
| 814 |
+
rgb_x, rgb_y, rgb_error, rgb_ties = _locate_unedited(
|
| 815 |
+
background, unedited_rect, interrupt, respect_background_alpha=False)
|
| 816 |
+
rgb_region = background[rgb_y:rgb_y + unedited_rect.shape[0],
|
| 817 |
+
rgb_x:rgb_x + unedited_rect.shape[1]]
|
| 818 |
+
visible_evidence = int(np.count_nonzero(rgb_region[..., 3] > _ALPHA_EPSILON))
|
| 819 |
+
minimum_evidence = min(16, unedited_rect.shape[0] * unedited_rect.shape[1])
|
| 820 |
+
rgb_texture = float(np.mean(np.var(unedited_rect[..., :3].astype(np.float64), axis=(0, 1))))
|
| 821 |
+
trustworthy_rgb = visible_evidence >= minimum_evidence and rgb_texture > 1.0e-10
|
| 822 |
+
if (rgb_error <= _RGB_COPY_ERROR_TOLERANCE and rgb_error < crop_error
|
| 823 |
+
and trustworthy_rgb):
|
| 824 |
+
crop_notes.append(
|
| 825 |
+
"Alpha-aware candidate (%d,%d) scored %.8f/200. A near-exact RGB copy instead matched (%d,%d)."
|
| 826 |
+
% (crop_x, crop_y, 200.0 * (1.0 - crop_error), rgb_x, rgb_y))
|
| 827 |
+
crop_notes.append(
|
| 828 |
+
"Crop alpha may be missing: connect the crop's matching Load Image MASK. "
|
| 829 |
+
"RGB fallback applies to crop location only; overlay output still needs its own real alpha/MASK.")
|
| 830 |
+
crop_x, crop_y, crop_error, ties = rgb_x, rgb_y, rgb_error, rgb_ties
|
| 831 |
+
crop_match_mode = "verified near-exact RGB-copy fallback"
|
| 832 |
+
elif rgb_error <= _RGB_COPY_ERROR_TOLERANCE and not trustworthy_rgb:
|
| 833 |
+
crop_notes.append(
|
| 834 |
+
"An RGB-only candidate had insufficient visible/texture evidence and was not accepted. "
|
| 835 |
+
"Hidden RGB in transparent pixels cannot establish a reliable visible match.")
|
| 836 |
+
crop_similarity = 200.0 * (1.0 - crop_error)
|
| 837 |
+
if crop_similarity < _MIN_CROP_SIMILARITY:
|
| 838 |
+
# Solver convergence is not evidence of correct crop localization.
|
| 839 |
+
# Return an explicit failure through the existing status socket and
|
| 840 |
+
# preserve background instead of pasting a wrong result somewhere.
|
| 841 |
+
status = "\n".join([
|
| 842 |
+
"LOCATION FAILED: original crop was not located reliably. No overlay was pasted.",
|
| 843 |
+
"Closest candidate: top_left=(%d,%d), size=%dx%d, similarity=%.8f/200; required >= %.4f/200."
|
| 844 |
+
% (crop_x, crop_y, unedited_rect.shape[1], unedited_rect.shape[0], crop_similarity,
|
| 845 |
+
_MIN_CROP_SIMILARITY),
|
| 846 |
+
"The unedited crop should match its original background near 200/200. "
|
| 847 |
+
"Check image wiring, crop resizing/flipping, and corresponding MASK/alpha inputs.",
|
| 848 |
+
"Output coordinates=-1 and dimensions=0 indicate no accepted placement.",
|
| 849 |
+
input_diagnostics, *crop_notes,
|
| 850 |
+
])
|
| 851 |
+
progress(100)
|
| 852 |
+
_LOG.warning("Salia: rejected unverified crop at (%d,%d), similarity %.8f/200", crop_x, crop_y, crop_similarity)
|
| 853 |
+
return {"composite": background.copy(), "overlay_canvas": np.zeros_like(background),
|
| 854 |
+
"status": status, "x": -1.0, "y": -1.0, "width": 0.0, "height": 0.0,
|
| 855 |
+
"center_x": -1.0, "center_y": -1.0, "similarity": 0.0,
|
| 856 |
+
"registration_similarity": 0.0, "robust_cost": None, "initial_robust_cost": None,
|
| 857 |
+
"converged": False, "accepted": False, "location_verified": False,
|
| 858 |
+
"trace": [], "parameters": np.zeros(4), "crop_x": crop_x, "crop_y": crop_y,
|
| 859 |
+
"crop_similarity": float(crop_similarity), "crop_match_mode": crop_match_mode}
|
| 860 |
crop_height, crop_width = unedited_rect.shape[:2]
|
| 861 |
center_x, center_y = crop_x + crop_width / 2.0, crop_y + crop_height / 2.0
|
| 862 |
width0, height0 = float(target_width or crop_width), float(target_height or crop_height)
|
|
|
|
| 886 |
lower[2:] = np.minimum(lower[2:], -1.0e-12)
|
| 887 |
upper[2:] = np.maximum(upper[2:], 1.0e-12)
|
| 888 |
parameters = np.zeros(4, dtype=np.float64)
|
| 889 |
+
trace, notices = [], list(crop_notes)
|
| 890 |
final_problem = None
|
| 891 |
last_result = None
|
| 892 |
total_calls = 0
|
|
|
|
| 961 |
% (state, "solver converged numerically" if converged else "inspect solver notes"),
|
| 962 |
"Crop: top_left=(%d,%d), size=%dx%d, similarity=%.8f/200"
|
| 963 |
% (crop_x, crop_y, crop_width, crop_height, 200.0 * (1.0 - crop_error)),
|
| 964 |
+
"Crop match mode: " + crop_match_mode,
|
| 965 |
+
input_diagnostics,
|
| 966 |
"Overlay: top_left=(%.8f,%.8f), size=(%.8f,%.8f), center=(%.8f,%.8f)"
|
| 967 |
% (x, y, width, height, center_x, center_y),
|
| 968 |
"Rendered similarity=%.8f/200; registration similarity=%.8f/200; robust cost %.10g -> %.10g"
|
|
|
|
| 982 |
"center_x": float(center_x), "center_y": float(center_y), "similarity": similarity,
|
| 983 |
"registration_similarity": registration_similarity, "robust_cost": fit_cost,
|
| 984 |
"initial_robust_cost": base_cost, "converged": converged, "accepted": accepted,
|
| 985 |
+
"trace": trace, "parameters": parameters.copy(), "crop_x": crop_x, "crop_y": crop_y,
|
| 986 |
+
"location_verified": True, "crop_similarity": float(crop_similarity),
|
| 987 |
+
"crop_match_mode": crop_match_mode}
|
| 988 |
|
| 989 |
|
| 990 |
class Salia_Auto_Align_Overlay:
|