Spaces:
Running on Zero
Running on Zero
File size: 12,980 Bytes
7dff04f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 | """Generative API clients for double-exposure separation."""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import List, Optional, Tuple
import numpy as np
from PIL import Image
from app.preprocessing import to_pil
DEFAULT_SEPARATION_MODEL = os.environ.get(
"REPLICATE_SEPARATION_MODEL",
"black-forest-labs/flux-dev",
)
DEMO_PERCENTILES = (35.0, 45.0, 50.0, 55.0, 65.0)
REPLICATE_STRENGTHS = (0.55, 0.65, 0.75)
@dataclass
class SeparationResult:
"""Output from a separation attempt."""
image_a: np.ndarray # float RGB [0, 1]
image_b: np.ndarray
method: str
message: str
candidate_id: str = "default"
scan_analysis: Optional["ScanAnalysis"] = None # WP-5.1 Fix 7: carry VLM analysis to avoid re-call
diagnostics: Optional[dict] = None # WP-6: structured per-source numbers (e.g. DIP init/best loss)
def _luminance(rgb: np.ndarray) -> np.ndarray:
return (
0.2126 * rgb[..., 0] + 0.7152 * rgb[..., 1] + 0.0722 * rgb[..., 2]
)
def _apply_hard_mask(base: np.ndarray, mask: np.ndarray, fill: float = 0.35) -> np.ndarray:
gray = np.mean(base, axis=-1, keepdims=True)
out = base * mask[..., np.newaxis] + gray * (~mask[..., np.newaxis]) * fill
return np.clip(out, 0.0, 1.0).astype(np.float32)
def _demo_separation_percentile(
positive_rgb: np.ndarray,
percentile: float,
candidate_id: str,
soft: bool = False,
) -> SeparationResult:
"""Split by luminance percentile; optional soft Gaussian boundary."""
lum = _luminance(positive_rgb)
threshold = float(np.percentile(lum, percentile))
if soft:
sigma = max(threshold * 0.15, 0.02)
weight_a = np.exp(-((lum - threshold) ** 2) / (2 * sigma**2))
weight_a = np.where(lum <= threshold, 1.0, weight_a)
weight_b = 1.0 - weight_a
image_a = np.clip(positive_rgb * weight_a[..., np.newaxis], 0.0, 1.0)
image_b = np.clip(positive_rgb * weight_b[..., np.newaxis], 0.0, 1.0)
strategy = f"soft_p{int(percentile)}"
else:
mask_a = lum <= threshold
image_a = _apply_hard_mask(positive_rgb, mask_a)
image_b = _apply_hard_mask(positive_rgb, ~mask_a)
strategy = f"hard_p{int(percentile)}"
return SeparationResult(
image_a=image_a.astype(np.float32),
image_b=image_b.astype(np.float32),
method=f"demo_{strategy}",
message="Demo heuristic separation (no API key).",
candidate_id=candidate_id,
)
def _demo_separation_spatial(positive_rgb: np.ndarray, axis: str) -> SeparationResult:
"""Split along horizontal or vertical midline with feathered blend."""
h, w = positive_rgb.shape[:2]
if axis == "horizontal":
coord = np.linspace(0, 1, h)[:, np.newaxis]
coord = np.broadcast_to(coord, (h, w))
cid = "spatial_h"
else:
coord = np.linspace(0, 1, w)[np.newaxis, :]
coord = np.broadcast_to(coord, (h, w))
cid = "spatial_v"
weight_a = np.clip(1.0 - np.abs(coord - 0.5) * 4.0, 0.0, 1.0)
weight_b = 1.0 - weight_a
image_a = np.clip(positive_rgb * weight_a[..., np.newaxis], 0.0, 1.0)
image_b = np.clip(positive_rgb * weight_b[..., np.newaxis], 0.0, 1.0)
return SeparationResult(
image_a=image_a.astype(np.float32),
image_b=image_b.astype(np.float32),
method=f"demo_{cid}",
message="Demo heuristic separation (no API key).",
candidate_id=cid,
)
def generate_demo_candidates(
positive_rgb: np.ndarray,
num_candidates: int = 3,
) -> List[SeparationResult]:
"""Generate diverse demo candidates without API access."""
pool: List[SeparationResult] = []
for p in DEMO_PERCENTILES:
pool.append(
_demo_separation_percentile(
positive_rgb, p, candidate_id=f"hard_p{int(p)}", soft=False
)
)
pool.append(
_demo_separation_percentile(
positive_rgb, 50.0, candidate_id="soft_p50", soft=True
)
)
pool.extend([
_demo_separation_spatial(positive_rgb, "horizontal"),
_demo_separation_spatial(positive_rgb, "vertical"),
])
return pool[: max(1, min(num_candidates, len(pool)))]
def _replicate_separation(
positive_rgb: np.ndarray,
model: str = DEFAULT_SEPARATION_MODEL,
prompt_strength: float = 0.65,
candidate_id: str = "replicate_0",
) -> SeparationResult:
"""Call Replicate for generative separation via image-to-image prompting."""
import replicate
pil = to_pil(positive_rgb)
prompt = (
"Separate this double-exposed photograph into the two distinct original "
"scenes. Recover clear, photorealistic details from each exposure."
)
output = replicate.run(
model,
input={
"prompt": prompt,
"image": pil,
"prompt_strength": prompt_strength,
"num_inference_steps": 28,
"guidance": 3.5,
},
)
if isinstance(output, list):
url = str(output[0])
else:
url = str(output)
from io import BytesIO
import urllib.request
with urllib.request.urlopen(url) as resp:
gen_img = Image.open(BytesIO(resp.read())).convert("RGB")
gen_rgb = np.asarray(gen_img, dtype=np.float32) / 255.0
if gen_rgb.shape[:2] != positive_rgb.shape[:2]:
gen_pil = gen_img.resize(
(positive_rgb.shape[1], positive_rgb.shape[0]),
Image.Resampling.LANCZOS,
)
gen_rgb = np.asarray(gen_pil, dtype=np.float32) / 255.0
lum_gen = _luminance(gen_rgb)
lum_orig = _luminance(positive_rgb)
lum_b = np.clip(lum_orig - lum_gen * 0.5, 0.0, 1.0)
scale = lum_b[..., np.newaxis] / np.clip(lum_orig[..., np.newaxis], 1e-4, 1.0)
image_b = np.clip(positive_rgb * scale, 0.0, 1.0).astype(np.float32)
return SeparationResult(
image_a=gen_rgb.astype(np.float32),
image_b=image_b,
method=f"replicate:{model}@s{prompt_strength:.2f}",
message=f"Generative separation via {model} (strength={prompt_strength:.2f})",
candidate_id=candidate_id,
)
def generate_replicate_candidates(
positive_rgb: np.ndarray,
num_candidates: int = 3,
model: str = DEFAULT_SEPARATION_MODEL,
) -> List[SeparationResult]:
"""Generate multiple Replicate separations with varied prompt strength."""
strengths = REPLICATE_STRENGTHS[: max(1, num_candidates)]
results: List[SeparationResult] = []
for i, strength in enumerate(strengths):
try:
results.append(
_replicate_separation(
positive_rgb,
model=model,
prompt_strength=strength,
candidate_id=f"replicate_s{int(strength * 100)}",
)
)
except Exception as exc:
results.append(
SeparationResult(
image_a=positive_rgb.copy(),
image_b=positive_rgb.copy(),
method="replicate_error",
message=f"Candidate failed: {exc}",
candidate_id=f"replicate_fail_{i}",
)
)
return results
def _append_one_demix(candidates, positive_rgb, h_total, confidence_mask, img2img, vlm, method):
"""WP-5.1 Fix 8 helper to dedup append logic."""
if h_total is None or confidence_mask is None:
return
try:
from app.demix import analyze_scan, residual_demix, DemixConfig
analysis = analyze_scan(positive_rgb, vlm=vlm)
cfg = DemixConfig(iterations=2, strength=0.55, use_instruct_edit=("instruct" in method))
d = residual_demix(positive_rgb, h_total, confidence_mask, img2img, analysis, cfg, method=method)
if "instruct" in method:
d.candidate_id = f"demix_instruct_k{analysis.k_judgment:.1f}_i{cfg.iterations}"
candidates.append(d)
except Exception:
pass
def _append_deep_prior(
candidates,
positive_rgb,
density,
log_exposure,
confidence_mask,
film_curve,
dip_policy=None,
):
"""WP-6 helper: append the Double-DIP candidate (soft-fail, lazy import).
Skipped silently when density/log_exposure are unavailable (same rule as demix:
never optimize against the legacy circular objective).
WP-13.1: dip_policy defaults to DEFAULT_POLICY (bench-safe); app passes APP_POLICY.
"""
if density is None or log_exposure is None or confidence_mask is None:
return
try:
from baselines.double_dip import double_dip_separate, DoubleDIPConfig
from scoring_policy import DEFAULT_POLICY
if film_curve is None:
from film_physics import get_film_curve
film_curve = get_film_curve("Generic")
cfg = DoubleDIPConfig(policy=dip_policy or DEFAULT_POLICY)
res = double_dip_separate(
positive_rgb, log_exposure, density, confidence_mask, film_curve, config=cfg
)
if res is not None:
candidates.append(res)
except Exception:
pass
def generate_candidates(
positive_rgb: np.ndarray,
num_candidates: int = 3,
api_token: Optional[str] = None,
model: str = DEFAULT_SEPARATION_MODEL,
h_total: Optional[np.ndarray] = None,
confidence_mask: Optional[np.ndarray] = None,
density: Optional[np.ndarray] = None,
log_exposure=None,
include_deep_prior: bool = False,
film_curve=None,
dip_policy=None,
) -> Tuple[List[SeparationResult], str]:
"""
Generate multiple separation candidates for ranking.
h_total / confidence_mask (WP-5): when provided, demix source is registered
(stub in demo; replicate+ instruct variant when token).
density / log_exposure / include_deep_prior / film_curve (WP-6): when the flag
is set and the density path is live, the Double-DIP source is registered.
film_curve should be the user's selected stock so DIP optimizes the right physics.
Returns:
Tuple of (candidate list, mode description string).
"""
num_candidates = max(1, min(int(num_candidates), 5))
token = api_token or os.environ.get("REPLICATE_API_TOKEN", "").strip()
if not token:
candidates = generate_demo_candidates(positive_rgb, num_candidates)
if h_total is not None and confidence_mask is not None:
# WP-5.1 Fix 4+8: demo uses helper, vlm=None
from app.demix import stub_cleanup
_append_one_demix(candidates, positive_rgb, h_total, confidence_mask, stub_cleanup, None, "demix_stub")
if include_deep_prior:
_append_deep_prior(
candidates, positive_rgb, density, log_exposure, confidence_mask, film_curve,
dip_policy=dip_policy,
)
return candidates, "demo"
os.environ["REPLICATE_API_TOKEN"] = token
try:
candidates = generate_replicate_candidates(positive_rgb, num_candidates, model)
if all(c.method == "replicate_error" for c in candidates):
raise RuntimeError("All Replicate candidates failed")
# WP-5.1 Fix 3+4+8: append via helper; gate instruct
if h_total is not None and confidence_mask is not None:
from app.demix import replicate_img2img as _rep_img2img
vlm = (lambda: __import__("app.demix", fromlist=["anthropic_vlm"]).anthropic_vlm if os.environ.get("ANTHROPIC_API_KEY") else None)()
_append_one_demix(candidates, positive_rgb, h_total, confidence_mask, _rep_img2img, vlm, "demix_replicate")
if num_candidates >= 3:
_append_one_demix(candidates, positive_rgb, h_total, confidence_mask, _rep_img2img, vlm, "demix_instruct")
if include_deep_prior:
_append_deep_prior(
candidates, positive_rgb, density, log_exposure, confidence_mask, film_curve,
dip_policy=dip_policy,
)
return candidates, "replicate"
except Exception as exc:
candidates = generate_demo_candidates(positive_rgb, num_candidates)
for c in candidates:
c.message = f"Replicate unavailable ({exc}). Using demo candidate."
if h_total is not None and confidence_mask is not None:
# WP-5.1 Fix 4+8: fallback demo via helper, vlm=None
from app.demix import stub_cleanup
_append_one_demix(candidates, positive_rgb, h_total, confidence_mask, stub_cleanup, None, "demix_stub")
if include_deep_prior:
_append_deep_prior(
candidates, positive_rgb, density, log_exposure, confidence_mask, film_curve,
dip_policy=dip_policy,
)
return candidates, "demo_fallback"
def result_to_pil_pair(result: SeparationResult) -> Tuple[Image.Image, Image.Image]:
"""Convert separation arrays to PIL images for display."""
return to_pil(result.image_a), to_pil(result.image_b)
|