Spaces:
Running on Zero
Running on Zero
File size: 4,085 Bytes
d2c5d46 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | import torch
from torch import nn
from diffcve.utils import instantiate_from_config, resolve_path
class FrozenOpenCLIPEmbedder(nn.Module):
def __init__(self, cond_model_path, arch="ViT-H-14", layer="penultimate", device="cpu"):
super().__init__()
if layer not in ("last", "penultimate"):
raise ValueError("OpenCLIP layer must be last or penultimate")
checkpoint_path = resolve_path(cond_model_path)
if not checkpoint_path.is_file():
raise FileNotFoundError(checkpoint_path)
import open_clip
model = open_clip.create_model(arch, device=torch.device("cpu"))
checkpoint = torch.load(checkpoint_path, map_location="cpu")
model.load_state_dict(checkpoint.get("state_dict", checkpoint), strict=True)
del model.visual
self.model = model.eval().requires_grad_(False).to(device)
self.layer_idx = 0 if layer == "last" else 1
self.tokenize = open_clip.tokenize
@torch.no_grad()
def forward(self, text):
tokens = self.tokenize(text).to(self.model.token_embedding.weight.device)
x = self.model.token_embedding(tokens) + self.model.positional_embedding
x = x.permute(1, 0, 2)
blocks = self.model.transformer.resblocks
for index, block in enumerate(blocks):
if index == len(blocks) - self.layer_idx:
break
x = block(x, attn_mask=self.model.attn_mask)
return self.model.ln_final(x.permute(1, 0, 2))
def encode(self, text):
return self(text)
def qp_interpolation_weights(qp, anchors):
anchors = sorted(anchors)
if not anchors:
raise ValueError("At least one text anchor is required")
if qp in anchors:
return {qp: 1.0}
if qp < anchors[0]:
return {anchors[0]: 1.0}
if qp > anchors[-1]:
return {anchors[-1]: 1.0}
for lower, upper in zip(anchors, anchors[1:]):
if lower < qp < upper:
weight = (qp - lower) / (upper - lower)
return {lower: 1.0 - weight, upper: weight}
def interpolate_text_conditions(anchors, qps):
conditions = {}
for qp in sorted(set(qps)):
weights = qp_interpolation_weights(qp, anchors)
reference = anchors[next(iter(weights))]
if len(weights) == 1:
conditions[qp] = reference
else:
combined = sum(anchors[key].float() * weight for key, weight in weights.items())
if not torch.isfinite(combined).all():
raise ValueError(f"QP {qp}: nonfinite interpolated text condition")
conditions[qp] = combined.to(dtype=reference.dtype)
return conditions
def load_text_conditions(path, encoder_config=None, prompts=None, qps=None):
if encoder_config is None:
if prompts is not None:
raise ValueError("text_prompts requires cond_stage_config")
conditions = torch.load(resolve_path(path), map_location="cpu")
else:
if not prompts or any(not isinstance(text, str) or not text.strip() for text in prompts.values()):
raise ValueError("text_prompts must map QPs to nonempty text strings")
encoder = instantiate_from_config(encoder_config)
with torch.no_grad():
conditions = {qp: encoder.encode([text]).detach().cpu() for qp, text in prompts.items()}
if not isinstance(conditions, dict) or not conditions:
raise ValueError("Text conditions must be a nonempty QP-to-embedding dictionary")
result = {}
for qp, embedding in conditions.items():
key = int(qp)
if key in result:
raise ValueError(f"Duplicate text condition for QP {key}")
if (not isinstance(embedding, torch.Tensor) or embedding.shape != (1, 77, 1024)
or not embedding.is_floating_point() or not torch.isfinite(embedding).all()):
raise ValueError(f"QP {key}: expected a finite floating-point embedding of shape [1, 77, 1024]")
result[key] = embedding
return result if qps is None else interpolate_text_conditions(result, qps)
|