Giulio Federico commited on
Commit
cd8decb
·
1 Parent(s): 19cb433

adding extern folder

Browse files
Files changed (3) hide show
  1. extern/__init__.py +0 -0
  2. extern/zero123.py +666 -0
  3. extern/zero123_official.py +894 -0
extern/__init__.py ADDED
File without changes
extern/zero123.py ADDED
@@ -0,0 +1,666 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright 2023 The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ import math
17
+ import warnings
18
+ from typing import Any, Callable, Dict, List, Optional, Union
19
+
20
+ import PIL
21
+ import torch
22
+ import torchvision.transforms.functional as TF
23
+ from diffusers.configuration_utils import ConfigMixin, FrozenDict, register_to_config
24
+ from diffusers.image_processor import VaeImageProcessor
25
+ from diffusers.models import AutoencoderKL, UNet2DConditionModel
26
+ from diffusers.models.modeling_utils import ModelMixin
27
+ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
28
+ from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
29
+ from diffusers.pipelines.stable_diffusion.safety_checker import (
30
+ StableDiffusionSafetyChecker,
31
+ )
32
+ from diffusers.schedulers import KarrasDiffusionSchedulers
33
+ from diffusers.utils import deprecate, is_accelerate_available, logging
34
+ from diffusers.utils.torch_utils import randn_tensor
35
+ from packaging import version
36
+ from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection
37
+
38
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
39
+
40
+
41
+ class CLIPCameraProjection(ModelMixin, ConfigMixin):
42
+ """
43
+ A Projection layer for CLIP embedding and camera embedding.
44
+
45
+ Parameters:
46
+ embedding_dim (`int`, *optional*, defaults to 768): The dimension of the model input `clip_embed`
47
+ additional_embeddings (`int`, *optional*, defaults to 4): The number of additional tokens appended to the
48
+ projected `hidden_states`. The actual length of the used `hidden_states` is `num_embeddings +
49
+ additional_embeddings`.
50
+ """
51
+
52
+ @register_to_config
53
+ def __init__(self, embedding_dim: int = 768, additional_embeddings: int = 4):
54
+ super().__init__()
55
+ self.embedding_dim = embedding_dim
56
+ self.additional_embeddings = additional_embeddings
57
+
58
+ self.input_dim = self.embedding_dim + self.additional_embeddings
59
+ self.output_dim = self.embedding_dim
60
+
61
+ self.proj = torch.nn.Linear(self.input_dim, self.output_dim)
62
+
63
+ def forward(
64
+ self,
65
+ embedding: torch.FloatTensor,
66
+ ):
67
+ """
68
+ The [`PriorTransformer`] forward method.
69
+
70
+ Args:
71
+ hidden_states (`torch.FloatTensor` of shape `(batch_size, input_dim)`):
72
+ The currently input embeddings.
73
+
74
+ Returns:
75
+ The output embedding projection (`torch.FloatTensor` of shape `(batch_size, output_dim)`).
76
+ """
77
+ proj_embedding = self.proj(embedding)
78
+ return proj_embedding
79
+
80
+
81
+ class Zero123Pipeline(DiffusionPipeline):
82
+ r"""
83
+ Pipeline to generate variations from an input image using Stable Diffusion.
84
+
85
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
86
+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
87
+
88
+ Args:
89
+ vae ([`AutoencoderKL`]):
90
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
91
+ image_encoder ([`CLIPVisionModelWithProjection`]):
92
+ Frozen CLIP image-encoder. Stable Diffusion Image Variation uses the vision portion of
93
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPVisionModelWithProjection),
94
+ specifically the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
95
+ unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
96
+ scheduler ([`SchedulerMixin`]):
97
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
98
+ [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
99
+ safety_checker ([`StableDiffusionSafetyChecker`]):
100
+ Classification module that estimates whether generated images could be considered offensive or harmful.
101
+ Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.
102
+ feature_extractor ([`CLIPImageProcessor`]):
103
+ Model that extracts features from generated images to be used as inputs for the `safety_checker`.
104
+ """
105
+ # TODO: feature_extractor is required to encode images (if they are in PIL format),
106
+ # we should give a descriptive message if the pipeline doesn't have one.
107
+ _optional_components = ["safety_checker"]
108
+
109
+ def __init__(
110
+ self,
111
+ vae: AutoencoderKL,
112
+ image_encoder: CLIPVisionModelWithProjection,
113
+ unet: UNet2DConditionModel,
114
+ scheduler: KarrasDiffusionSchedulers,
115
+ safety_checker: StableDiffusionSafetyChecker,
116
+ feature_extractor: CLIPImageProcessor,
117
+ clip_camera_projection: CLIPCameraProjection,
118
+ requires_safety_checker: bool = True,
119
+ ):
120
+ super().__init__()
121
+
122
+ if safety_checker is None and requires_safety_checker:
123
+ logger.warn(
124
+ f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"
125
+ " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"
126
+ " results in services or applications open to the public. Both the diffusers team and Hugging Face"
127
+ " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"
128
+ " it only for use-cases that involve analyzing network behavior or auditing its results. For more"
129
+ " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."
130
+ )
131
+
132
+ if safety_checker is not None and feature_extractor is None:
133
+ raise ValueError(
134
+ "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"
135
+ " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
136
+ )
137
+
138
+ is_unet_version_less_0_9_0 = hasattr(
139
+ unet.config, "_diffusers_version"
140
+ ) and version.parse(
141
+ version.parse(unet.config._diffusers_version).base_version
142
+ ) < version.parse(
143
+ "0.9.0.dev0"
144
+ )
145
+ is_unet_sample_size_less_64 = (
146
+ hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
147
+ )
148
+ if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:
149
+ deprecation_message = (
150
+ "The configuration file of the unet has set the default `sample_size` to smaller than"
151
+ " 64 which seems highly unlikely .If you're checkpoint is a fine-tuned version of any of the"
152
+ " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"
153
+ " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"
154
+ " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"
155
+ " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"
156
+ " in the config might lead to incorrect results in future versions. If you have downloaded this"
157
+ " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"
158
+ " the `unet/config.json` file"
159
+ )
160
+ deprecate(
161
+ "sample_size<64", "1.0.0", deprecation_message, standard_warn=False
162
+ )
163
+ new_config = dict(unet.config)
164
+ new_config["sample_size"] = 64
165
+ unet._internal_dict = FrozenDict(new_config)
166
+
167
+ self.register_modules(
168
+ vae=vae,
169
+ image_encoder=image_encoder,
170
+ unet=unet,
171
+ scheduler=scheduler,
172
+ safety_checker=safety_checker,
173
+ feature_extractor=feature_extractor,
174
+ clip_camera_projection=clip_camera_projection,
175
+ )
176
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
177
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
178
+ self.register_to_config(requires_safety_checker=requires_safety_checker)
179
+
180
+ def enable_sequential_cpu_offload(self, gpu_id=0):
181
+ r"""
182
+ Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet,
183
+ text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a
184
+ `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called.
185
+ """
186
+ if is_accelerate_available():
187
+ from accelerate import cpu_offload
188
+ else:
189
+ raise ImportError("Please install accelerate via `pip install accelerate`")
190
+
191
+ device = torch.device(f"cuda:{gpu_id}")
192
+
193
+ for cpu_offloaded_model in [
194
+ self.unet,
195
+ self.image_encoder,
196
+ self.vae,
197
+ self.safety_checker,
198
+ ]:
199
+ if cpu_offloaded_model is not None:
200
+ cpu_offload(cpu_offloaded_model, device)
201
+
202
+ @property
203
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._execution_device
204
+ def _execution_device(self):
205
+ r"""
206
+ Returns the device on which the pipeline's models will be executed. After calling
207
+ `pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module
208
+ hooks.
209
+ """
210
+ if not hasattr(self.unet, "_hf_hook"):
211
+ return self.device
212
+ for module in self.unet.modules():
213
+ if (
214
+ hasattr(module, "_hf_hook")
215
+ and hasattr(module._hf_hook, "execution_device")
216
+ and module._hf_hook.execution_device is not None
217
+ ):
218
+ return torch.device(module._hf_hook.execution_device)
219
+ return self.device
220
+
221
+ def _encode_image(
222
+ self,
223
+ image,
224
+ elevation,
225
+ azimuth,
226
+ distance,
227
+ device,
228
+ num_images_per_prompt,
229
+ do_classifier_free_guidance,
230
+ clip_image_embeddings=None,
231
+ image_camera_embeddings=None,
232
+ ):
233
+ dtype = next(self.image_encoder.parameters()).dtype
234
+
235
+ if image_camera_embeddings is None:
236
+ if image is None:
237
+ assert clip_image_embeddings is not None
238
+ image_embeddings = clip_image_embeddings.to(device=device, dtype=dtype)
239
+ else:
240
+ if not isinstance(image, torch.Tensor):
241
+ image = self.feature_extractor(
242
+ images=image, return_tensors="pt"
243
+ ).pixel_values
244
+
245
+ image = image.to(device=device, dtype=dtype)
246
+ image_embeddings = self.image_encoder(image).image_embeds
247
+ image_embeddings = image_embeddings.unsqueeze(1)
248
+
249
+ bs_embed, seq_len, _ = image_embeddings.shape
250
+
251
+ if isinstance(elevation, float):
252
+ elevation = torch.as_tensor(
253
+ [elevation] * bs_embed, dtype=dtype, device=device
254
+ )
255
+ if isinstance(azimuth, float):
256
+ azimuth = torch.as_tensor(
257
+ [azimuth] * bs_embed, dtype=dtype, device=device
258
+ )
259
+ if isinstance(distance, float):
260
+ distance = torch.as_tensor(
261
+ [distance] * bs_embed, dtype=dtype, device=device
262
+ )
263
+
264
+ camera_embeddings = torch.stack(
265
+ [
266
+ torch.deg2rad(elevation),
267
+ torch.sin(torch.deg2rad(azimuth)),
268
+ torch.cos(torch.deg2rad(azimuth)),
269
+ distance,
270
+ ],
271
+ dim=-1,
272
+ )[:, None, :]
273
+
274
+ image_embeddings = torch.cat([image_embeddings, camera_embeddings], dim=-1)
275
+
276
+ # project (image, camera) embeddings to the same dimension as clip embeddings
277
+ image_embeddings = self.clip_camera_projection(image_embeddings)
278
+ else:
279
+ image_embeddings = image_camera_embeddings.to(device=device, dtype=dtype)
280
+ bs_embed, seq_len, _ = image_embeddings.shape
281
+
282
+ # duplicate image embeddings for each generation per prompt, using mps friendly method
283
+ image_embeddings = image_embeddings.repeat(1, num_images_per_prompt, 1)
284
+ image_embeddings = image_embeddings.view(
285
+ bs_embed * num_images_per_prompt, seq_len, -1
286
+ )
287
+
288
+ if do_classifier_free_guidance:
289
+ negative_prompt_embeds = torch.zeros_like(image_embeddings)
290
+
291
+ # For classifier free guidance, we need to do two forward passes.
292
+ # Here we concatenate the unconditional and text embeddings into a single batch
293
+ # to avoid doing two forward passes
294
+ image_embeddings = torch.cat([negative_prompt_embeds, image_embeddings])
295
+
296
+ return image_embeddings
297
+
298
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker
299
+ def run_safety_checker(self, image, device, dtype):
300
+ if self.safety_checker is None:
301
+ has_nsfw_concept = None
302
+ else:
303
+ if torch.is_tensor(image):
304
+ feature_extractor_input = self.image_processor.postprocess(
305
+ image, output_type="pil"
306
+ )
307
+ else:
308
+ feature_extractor_input = self.image_processor.numpy_to_pil(image)
309
+ safety_checker_input = self.feature_extractor(
310
+ feature_extractor_input, return_tensors="pt"
311
+ ).to(device)
312
+ image, has_nsfw_concept = self.safety_checker(
313
+ images=image, clip_input=safety_checker_input.pixel_values.to(dtype)
314
+ )
315
+ return image, has_nsfw_concept
316
+
317
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents
318
+ def decode_latents(self, latents):
319
+ warnings.warn(
320
+ "The decode_latents method is deprecated and will be removed in a future version. Please"
321
+ " use VaeImageProcessor instead",
322
+ FutureWarning,
323
+ )
324
+ latents = 1 / self.vae.config.scaling_factor * latents
325
+ image = self.vae.decode(latents, return_dict=False)[0]
326
+ image = (image / 2 + 0.5).clamp(0, 1)
327
+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
328
+ image = image.cpu().permute(0, 2, 3, 1).float().numpy()
329
+ return image
330
+
331
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
332
+ def prepare_extra_step_kwargs(self, generator, eta):
333
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
334
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
335
+ # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
336
+ # and should be between [0, 1]
337
+
338
+ accepts_eta = "eta" in set(
339
+ inspect.signature(self.scheduler.step).parameters.keys()
340
+ )
341
+ extra_step_kwargs = {}
342
+ if accepts_eta:
343
+ extra_step_kwargs["eta"] = eta
344
+
345
+ # check if the scheduler accepts generator
346
+ accepts_generator = "generator" in set(
347
+ inspect.signature(self.scheduler.step).parameters.keys()
348
+ )
349
+ if accepts_generator:
350
+ extra_step_kwargs["generator"] = generator
351
+ return extra_step_kwargs
352
+
353
+ def check_inputs(self, image, height, width, callback_steps):
354
+ # TODO: check image size or adjust image size to (height, width)
355
+
356
+ if height % 8 != 0 or width % 8 != 0:
357
+ raise ValueError(
358
+ f"`height` and `width` have to be divisible by 8 but are {height} and {width}."
359
+ )
360
+
361
+ if (callback_steps is None) or (
362
+ callback_steps is not None
363
+ and (not isinstance(callback_steps, int) or callback_steps <= 0)
364
+ ):
365
+ raise ValueError(
366
+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
367
+ f" {type(callback_steps)}."
368
+ )
369
+
370
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
371
+ def prepare_latents(
372
+ self,
373
+ batch_size,
374
+ num_channels_latents,
375
+ height,
376
+ width,
377
+ dtype,
378
+ device,
379
+ generator,
380
+ latents=None,
381
+ ):
382
+ shape = (
383
+ batch_size,
384
+ num_channels_latents,
385
+ height // self.vae_scale_factor,
386
+ width // self.vae_scale_factor,
387
+ )
388
+ if isinstance(generator, list) and len(generator) != batch_size:
389
+ raise ValueError(
390
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
391
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
392
+ )
393
+
394
+ if latents is None:
395
+ latents = randn_tensor(
396
+ shape, generator=generator, device=device, dtype=dtype
397
+ )
398
+ else:
399
+ latents = latents.to(device)
400
+
401
+ # scale the initial noise by the standard deviation required by the scheduler
402
+ latents = latents * self.scheduler.init_noise_sigma
403
+ return latents
404
+
405
+ def _get_latent_model_input(
406
+ self,
407
+ latents: torch.FloatTensor,
408
+ image: Optional[
409
+ Union[PIL.Image.Image, List[PIL.Image.Image], torch.FloatTensor]
410
+ ],
411
+ num_images_per_prompt: int,
412
+ do_classifier_free_guidance: bool,
413
+ image_latents: Optional[torch.FloatTensor] = None,
414
+ ):
415
+ if isinstance(image, PIL.Image.Image):
416
+ image_pt = TF.to_tensor(image).unsqueeze(0).to(latents)
417
+ elif isinstance(image, list):
418
+ image_pt = torch.stack([TF.to_tensor(img) for img in image], dim=0).to(
419
+ latents
420
+ )
421
+ elif isinstance(image, torch.Tensor):
422
+ image_pt = image
423
+ else:
424
+ image_pt = None
425
+
426
+ if image_pt is None:
427
+ assert image_latents is not None
428
+ image_pt = image_latents.repeat_interleave(num_images_per_prompt, dim=0)
429
+ else:
430
+ image_pt = image_pt * 2.0 - 1.0 # scale to [-1, 1]
431
+ # FIXME: encoded latents should be multiplied with self.vae.config.scaling_factor
432
+ # but zero123 was not trained this way
433
+ image_pt = self.vae.encode(image_pt).latent_dist.mode()
434
+ image_pt = image_pt.repeat_interleave(num_images_per_prompt, dim=0)
435
+ if do_classifier_free_guidance:
436
+ latent_model_input = torch.cat(
437
+ [
438
+ torch.cat([latents, latents], dim=0),
439
+ torch.cat([torch.zeros_like(image_pt), image_pt], dim=0),
440
+ ],
441
+ dim=1,
442
+ )
443
+ else:
444
+ latent_model_input = torch.cat([latents, image_pt], dim=1)
445
+
446
+ return latent_model_input
447
+
448
+ @torch.no_grad()
449
+ def __call__(
450
+ self,
451
+ image: Optional[
452
+ Union[PIL.Image.Image, List[PIL.Image.Image], torch.FloatTensor]
453
+ ] = None,
454
+ elevation: Optional[Union[float, torch.FloatTensor]] = None,
455
+ azimuth: Optional[Union[float, torch.FloatTensor]] = None,
456
+ distance: Optional[Union[float, torch.FloatTensor]] = None,
457
+ height: Optional[int] = None,
458
+ width: Optional[int] = None,
459
+ num_inference_steps: int = 50,
460
+ guidance_scale: float = 3.0,
461
+ num_images_per_prompt: int = 1,
462
+ eta: float = 0.0,
463
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
464
+ latents: Optional[torch.FloatTensor] = None,
465
+ clip_image_embeddings: Optional[torch.FloatTensor] = None,
466
+ image_camera_embeddings: Optional[torch.FloatTensor] = None,
467
+ image_latents: Optional[torch.FloatTensor] = None,
468
+ output_type: Optional[str] = "pil",
469
+ return_dict: bool = True,
470
+ callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
471
+ callback_steps: int = 1,
472
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
473
+ ):
474
+ r"""
475
+ Function invoked when calling the pipeline for generation.
476
+
477
+ Args:
478
+ image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`):
479
+ The image or images to guide the image generation. If you provide a tensor, it needs to comply with the
480
+ configuration of
481
+ [this](https://huggingface.co/lambdalabs/sd-image-variations-diffusers/blob/main/feature_extractor/preprocessor_config.json)
482
+ `CLIPImageProcessor`
483
+ height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
484
+ The height in pixels of the generated image.
485
+ width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
486
+ The width in pixels of the generated image.
487
+ num_inference_steps (`int`, *optional*, defaults to 50):
488
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
489
+ expense of slower inference.
490
+ guidance_scale (`float`, *optional*, defaults to 7.5):
491
+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
492
+ `guidance_scale` is defined as `w` of equation 2. of [Imagen
493
+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
494
+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
495
+ usually at the expense of lower image quality.
496
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
497
+ The number of images to generate per prompt.
498
+ eta (`float`, *optional*, defaults to 0.0):
499
+ Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
500
+ [`schedulers.DDIMScheduler`], will be ignored for others.
501
+ generator (`torch.Generator`, *optional*):
502
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
503
+ to make generation deterministic.
504
+ latents (`torch.FloatTensor`, *optional*):
505
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
506
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
507
+ tensor will ge generated by sampling using the supplied random `generator`.
508
+ output_type (`str`, *optional*, defaults to `"pil"`):
509
+ The output format of the generate image. Choose between
510
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
511
+ return_dict (`bool`, *optional*, defaults to `True`):
512
+ Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
513
+ plain tuple.
514
+ callback (`Callable`, *optional*):
515
+ A function that will be called every `callback_steps` steps during inference. The function will be
516
+ called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
517
+ callback_steps (`int`, *optional*, defaults to 1):
518
+ The frequency at which the `callback` function will be called. If not specified, the callback will be
519
+ called at every step.
520
+
521
+ Returns:
522
+ [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
523
+ [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
524
+ When returning a tuple, the first element is a list with the generated images, and the second element is a
525
+ list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
526
+ (nsfw) content, according to the `safety_checker`.
527
+ """
528
+ # 0. Default height and width to unet
529
+ height = height or self.unet.config.sample_size * self.vae_scale_factor
530
+ width = width or self.unet.config.sample_size * self.vae_scale_factor
531
+
532
+ # 1. Check inputs. Raise error if not correct
533
+ # TODO: check input elevation, azimuth, and distance
534
+ # TODO: check image, clip_image_embeddings, image_latents
535
+ self.check_inputs(image, height, width, callback_steps)
536
+
537
+ # 2. Define call parameters
538
+ if isinstance(image, PIL.Image.Image):
539
+ batch_size = 1
540
+ elif isinstance(image, list):
541
+ batch_size = len(image)
542
+ elif isinstance(image, torch.Tensor):
543
+ batch_size = image.shape[0]
544
+ else:
545
+ assert image_latents is not None
546
+ assert (
547
+ clip_image_embeddings is not None or image_camera_embeddings is not None
548
+ )
549
+ batch_size = image_latents.shape[0]
550
+
551
+ device = self._execution_device
552
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
553
+ # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
554
+ # corresponds to doing no classifier free guidance.
555
+ do_classifier_free_guidance = guidance_scale > 1.0
556
+
557
+ # 3. Encode input image
558
+ if isinstance(image, PIL.Image.Image) or isinstance(image, list):
559
+ pil_image = image
560
+ elif isinstance(image, torch.Tensor):
561
+ pil_image = [TF.to_pil_image(image[i]) for i in range(image.shape[0])]
562
+ else:
563
+ pil_image = None
564
+ image_embeddings = self._encode_image(
565
+ pil_image,
566
+ elevation,
567
+ azimuth,
568
+ distance,
569
+ device,
570
+ num_images_per_prompt,
571
+ do_classifier_free_guidance,
572
+ clip_image_embeddings,
573
+ image_camera_embeddings,
574
+ )
575
+
576
+ # 4. Prepare timesteps
577
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
578
+ timesteps = self.scheduler.timesteps
579
+
580
+ # 5. Prepare latent variables
581
+ # num_channels_latents = self.unet.config.in_channels
582
+ num_channels_latents = 4 # FIXME: hard-coded
583
+ latents = self.prepare_latents(
584
+ batch_size * num_images_per_prompt,
585
+ num_channels_latents,
586
+ height,
587
+ width,
588
+ image_embeddings.dtype,
589
+ device,
590
+ generator,
591
+ latents,
592
+ )
593
+
594
+ # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
595
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
596
+
597
+ # 7. Denoising loop
598
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
599
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
600
+ for i, t in enumerate(timesteps):
601
+ # expand the latents if we are doing classifier free guidance
602
+ latent_model_input = self._get_latent_model_input(
603
+ latents,
604
+ image,
605
+ num_images_per_prompt,
606
+ do_classifier_free_guidance,
607
+ image_latents,
608
+ )
609
+ latent_model_input = self.scheduler.scale_model_input(
610
+ latent_model_input, t
611
+ )
612
+
613
+ # predict the noise residual
614
+ noise_pred = self.unet(
615
+ latent_model_input,
616
+ t,
617
+ encoder_hidden_states=image_embeddings,
618
+ cross_attention_kwargs=cross_attention_kwargs,
619
+ ).sample
620
+
621
+ # perform guidance
622
+ if do_classifier_free_guidance:
623
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
624
+ noise_pred = noise_pred_uncond + guidance_scale * (
625
+ noise_pred_text - noise_pred_uncond
626
+ )
627
+
628
+ # compute the previous noisy sample x_t -> x_t-1
629
+ latents = self.scheduler.step(
630
+ noise_pred, t, latents, **extra_step_kwargs
631
+ ).prev_sample
632
+
633
+ # call the callback, if provided
634
+ if i == len(timesteps) - 1 or (
635
+ (i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0
636
+ ):
637
+ progress_bar.update()
638
+ if callback is not None and i % callback_steps == 0:
639
+ callback(i, t, latents)
640
+
641
+ if not output_type == "latent":
642
+ image = self.vae.decode(
643
+ latents / self.vae.config.scaling_factor, return_dict=False
644
+ )[0]
645
+ image, has_nsfw_concept = self.run_safety_checker(
646
+ image, device, image_embeddings.dtype
647
+ )
648
+ else:
649
+ image = latents
650
+ has_nsfw_concept = None
651
+
652
+ if has_nsfw_concept is None:
653
+ do_denormalize = [True] * image.shape[0]
654
+ else:
655
+ do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]
656
+
657
+ image = self.image_processor.postprocess(
658
+ image, output_type=output_type, do_denormalize=do_denormalize
659
+ )
660
+
661
+ if not return_dict:
662
+ return (image, has_nsfw_concept)
663
+
664
+ return StableDiffusionPipelineOutput(
665
+ images=image, nsfw_content_detected=has_nsfw_concept
666
+ )
extern/zero123_official.py ADDED
@@ -0,0 +1,894 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # A diffuser version implementation of Zero1to3 (https://github.com/cvlab-columbia/zero123), ICCV 2023
2
+ # by Xin Kong
3
+
4
+ import inspect
5
+ import math
6
+ import sys
7
+ from typing import Any, Callable, Dict, List, Optional, Union
8
+
9
+ import einops
10
+ import kornia
11
+ import numpy as np
12
+ import PIL
13
+ import torch
14
+ import torch.nn.functional as F
15
+ from diffusers import AutoencoderKL, DiffusionPipeline, UNet2DConditionModel
16
+ from diffusers.configuration_utils import ConfigMixin, FrozenDict
17
+ from diffusers.models.modeling_utils import ModelMixin
18
+ from diffusers.pipelines.stable_diffusion import (
19
+ StableDiffusionPipelineOutput,
20
+ StableDiffusionSafetyChecker,
21
+ )
22
+ from diffusers.schedulers import KarrasDiffusionSchedulers
23
+ from diffusers.utils import (
24
+ deprecate,
25
+ is_accelerate_available,
26
+ is_accelerate_version,
27
+ logging,
28
+ replace_example_docstring,
29
+ )
30
+ from diffusers.utils.torch_utils import randn_tensor
31
+
32
+ from packaging import version
33
+ from transformers import (
34
+ CLIPFeatureExtractor,
35
+ CLIPTextModel,
36
+ CLIPTokenizer,
37
+ CLIPVisionModelWithProjection,
38
+ )
39
+
40
+ # from ...configuration_utils import FrozenDict
41
+ # from ...models import AutoencoderKL, UNet2DConditionModel
42
+ # from ...schedulers import KarrasDiffusionSchedulers
43
+ # from ...utils import (
44
+ # deprecate,
45
+ # is_accelerate_available,
46
+ # is_accelerate_version,
47
+ # logging,
48
+ # randn_tensor,
49
+ # replace_example_docstring,
50
+ # )
51
+ # from ..pipeline_utils import DiffusionPipeline
52
+ # from . import StableDiffusionPipelineOutput
53
+ # from .safety_checker import StableDiffusionSafetyChecker
54
+
55
+
56
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
57
+ # TODO
58
+ EXAMPLE_DOC_STRING = """
59
+ Examples:
60
+ ```py
61
+ >>> import torch
62
+ >>> from diffusers import StableDiffusionPipeline
63
+
64
+ >>> pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16)
65
+ >>> pipe = pipe.to("cuda")
66
+
67
+ >>> prompt = "a photo of an astronaut riding a horse on mars"
68
+ >>> image = pipe(prompt).images[0]
69
+ ```
70
+ """
71
+
72
+
73
+ class CCProjection(ModelMixin, ConfigMixin):
74
+ def __init__(self, in_channel=772, out_channel=768):
75
+ super().__init__()
76
+ self.in_channel = in_channel
77
+ self.out_channel = out_channel
78
+ self.projection = torch.nn.Linear(in_channel, out_channel)
79
+
80
+ def forward(self, x):
81
+ return self.projection(x)
82
+
83
+
84
+ class Zero1to3StableDiffusionPipeline(DiffusionPipeline):
85
+ r"""
86
+ Pipeline for single view conditioned novel view generation using Zero1to3.
87
+
88
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
89
+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
90
+
91
+ Args:
92
+ vae ([`AutoencoderKL`]):
93
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
94
+ image_encoder ([`CLIPVisionModelWithProjection`]):
95
+ Frozen CLIP image-encoder. Stable Diffusion Image Variation uses the vision portion of
96
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPVisionModelWithProjection),
97
+ specifically the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
98
+ tokenizer (`CLIPTokenizer`):
99
+ Tokenizer of class
100
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer).
101
+ unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents.
102
+ scheduler ([`SchedulerMixin`]):
103
+ A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of
104
+ [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`].
105
+ safety_checker ([`StableDiffusionSafetyChecker`]):
106
+ Classification module that estimates whether generated images could be considered offensive or harmful.
107
+ Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details.
108
+ feature_extractor ([`CLIPFeatureExtractor`]):
109
+ Model that extracts features from generated images to be used as inputs for the `safety_checker`.
110
+ """
111
+ _optional_components = ["safety_checker", "feature_extractor"]
112
+
113
+ def __init__(
114
+ self,
115
+ vae: AutoencoderKL,
116
+ image_encoder: CLIPVisionModelWithProjection,
117
+ unet: UNet2DConditionModel,
118
+ scheduler: KarrasDiffusionSchedulers,
119
+ safety_checker: StableDiffusionSafetyChecker,
120
+ feature_extractor: CLIPFeatureExtractor,
121
+ cc_projection: CCProjection,
122
+ requires_safety_checker: bool = True,
123
+ ):
124
+ super().__init__()
125
+
126
+ if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:
127
+ deprecation_message = (
128
+ f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"
129
+ f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "
130
+ "to update the config accordingly as leaving `steps_offset` might led to incorrect results"
131
+ " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub,"
132
+ " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`"
133
+ " file"
134
+ )
135
+ deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False)
136
+ new_config = dict(scheduler.config)
137
+ new_config["steps_offset"] = 1
138
+ scheduler._internal_dict = FrozenDict(new_config)
139
+
140
+ if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:
141
+ deprecation_message = (
142
+ f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."
143
+ " `clip_sample` should be set to False in the configuration file. Please make sure to update the"
144
+ " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in"
145
+ " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very"
146
+ " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file"
147
+ )
148
+ deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False)
149
+ new_config = dict(scheduler.config)
150
+ new_config["clip_sample"] = False
151
+ scheduler._internal_dict = FrozenDict(new_config)
152
+
153
+ if safety_checker is None and requires_safety_checker:
154
+ logger.warning(
155
+ f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"
156
+ " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"
157
+ " results in services or applications open to the public. Both the diffusers team and Hugging Face"
158
+ " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"
159
+ " it only for use-cases that involve analyzing network behavior or auditing its results. For more"
160
+ " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."
161
+ )
162
+
163
+ if safety_checker is not None and feature_extractor is None:
164
+ raise ValueError(
165
+ "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"
166
+ " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
167
+ )
168
+
169
+ is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(
170
+ version.parse(unet.config._diffusers_version).base_version
171
+ ) < version.parse("0.9.0.dev0")
172
+ is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
173
+ if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:
174
+ deprecation_message = (
175
+ "The configuration file of the unet has set the default `sample_size` to smaller than"
176
+ " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the"
177
+ " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-"
178
+ " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5"
179
+ " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the"
180
+ " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`"
181
+ " in the config might lead to incorrect results in future versions. If you have downloaded this"
182
+ " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for"
183
+ " the `unet/config.json` file"
184
+ )
185
+ deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False)
186
+ new_config = dict(unet.config)
187
+ new_config["sample_size"] = 64
188
+ unet._internal_dict = FrozenDict(new_config)
189
+
190
+ self.register_modules(
191
+ vae=vae,
192
+ image_encoder=image_encoder,
193
+ unet=unet,
194
+ scheduler=scheduler,
195
+ safety_checker=safety_checker,
196
+ feature_extractor=feature_extractor,
197
+ cc_projection=cc_projection,
198
+ )
199
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
200
+ self.register_to_config(requires_safety_checker=requires_safety_checker)
201
+ # self.model_mode = None
202
+
203
+ def enable_vae_slicing(self):
204
+ r"""
205
+ Enable sliced VAE decoding.
206
+
207
+ When this option is enabled, the VAE will split the input tensor in slices to compute decoding in several
208
+ steps. This is useful to save some memory and allow larger batch sizes.
209
+ """
210
+ self.vae.enable_slicing()
211
+
212
+ def disable_vae_slicing(self):
213
+ r"""
214
+ Disable sliced VAE decoding. If `enable_vae_slicing` was previously invoked, this method will go back to
215
+ computing decoding in one step.
216
+ """
217
+ self.vae.disable_slicing()
218
+
219
+ def enable_vae_tiling(self):
220
+ r"""
221
+ Enable tiled VAE decoding.
222
+
223
+ When this option is enabled, the VAE will split the input tensor into tiles to compute decoding and encoding in
224
+ several steps. This is useful to save a large amount of memory and to allow the processing of larger images.
225
+ """
226
+ self.vae.enable_tiling()
227
+
228
+ def disable_vae_tiling(self):
229
+ r"""
230
+ Disable tiled VAE decoding. If `enable_vae_tiling` was previously invoked, this method will go back to
231
+ computing decoding in one step.
232
+ """
233
+ self.vae.disable_tiling()
234
+
235
+ def enable_sequential_cpu_offload(self, gpu_id=0):
236
+ r"""
237
+ Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet,
238
+ text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a
239
+ `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called.
240
+ Note that offloading happens on a submodule basis. Memory savings are higher than with
241
+ `enable_model_cpu_offload`, but performance is lower.
242
+ """
243
+ if is_accelerate_available() and is_accelerate_version(">=", "0.14.0"):
244
+ from accelerate import cpu_offload
245
+ else:
246
+ raise ImportError("`enable_sequential_cpu_offload` requires `accelerate v0.14.0` or higher")
247
+
248
+ device = torch.device(f"cuda:{gpu_id}")
249
+
250
+ if self.device.type != "cpu":
251
+ self.to("cpu", silence_dtype_warnings=True)
252
+ torch.cuda.empty_cache() # otherwise we don't see the memory savings (but they probably exist)
253
+
254
+ for cpu_offloaded_model in [self.unet, self.text_encoder, self.vae]:
255
+ cpu_offload(cpu_offloaded_model, device)
256
+
257
+ if self.safety_checker is not None:
258
+ cpu_offload(self.safety_checker, execution_device=device, offload_buffers=True)
259
+
260
+ def enable_model_cpu_offload(self, gpu_id=0):
261
+ r"""
262
+ Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared
263
+ to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward`
264
+ method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with
265
+ `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`.
266
+ """
267
+ if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"):
268
+ from accelerate import cpu_offload_with_hook
269
+ else:
270
+ raise ImportError("`enable_model_offload` requires `accelerate v0.17.0` or higher.")
271
+
272
+ device = torch.device(f"cuda:{gpu_id}")
273
+
274
+ if self.device.type != "cpu":
275
+ self.to("cpu", silence_dtype_warnings=True)
276
+ torch.cuda.empty_cache() # otherwise we don't see the memory savings (but they probably exist)
277
+
278
+ hook = None
279
+ for cpu_offloaded_model in [self.text_encoder, self.unet, self.vae]:
280
+ _, hook = cpu_offload_with_hook(cpu_offloaded_model, device, prev_module_hook=hook)
281
+
282
+ if self.safety_checker is not None:
283
+ _, hook = cpu_offload_with_hook(self.safety_checker, device, prev_module_hook=hook)
284
+
285
+ # We'll offload the last model manually.
286
+ self.final_offload_hook = hook
287
+
288
+ @property
289
+ def _execution_device(self):
290
+ r"""
291
+ Returns the device on which the pipeline's models will be executed. After calling
292
+ `pipeline.enable_sequential_cpu_offload()` the execution device can only be inferred from Accelerate's module
293
+ hooks.
294
+ """
295
+ if not hasattr(self.unet, "_hf_hook"):
296
+ return self.device
297
+ for module in self.unet.modules():
298
+ if (
299
+ hasattr(module, "_hf_hook")
300
+ and hasattr(module._hf_hook, "execution_device")
301
+ and module._hf_hook.execution_device is not None
302
+ ):
303
+ return torch.device(module._hf_hook.execution_device)
304
+ return self.device
305
+
306
+ def _encode_prompt(
307
+ self,
308
+ prompt,
309
+ device,
310
+ num_images_per_prompt,
311
+ do_classifier_free_guidance,
312
+ negative_prompt=None,
313
+ prompt_embeds: Optional[torch.FloatTensor] = None,
314
+ negative_prompt_embeds: Optional[torch.FloatTensor] = None,
315
+ ):
316
+ r"""
317
+ Encodes the prompt into text encoder hidden states.
318
+
319
+ Args:
320
+ prompt (`str` or `List[str]`, *optional*):
321
+ prompt to be encoded
322
+ device: (`torch.device`):
323
+ torch device
324
+ num_images_per_prompt (`int`):
325
+ number of images that should be generated per prompt
326
+ do_classifier_free_guidance (`bool`):
327
+ whether to use classifier free guidance or not
328
+ negative_prompt (`str` or `List[str]`, *optional*):
329
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
330
+ `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.
331
+ Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).
332
+ prompt_embeds (`torch.FloatTensor`, *optional*):
333
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
334
+ provided, text embeddings will be generated from `prompt` input argument.
335
+ negative_prompt_embeds (`torch.FloatTensor`, *optional*):
336
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
337
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
338
+ argument.
339
+ """
340
+ if prompt is not None and isinstance(prompt, str):
341
+ batch_size = 1
342
+ elif prompt is not None and isinstance(prompt, list):
343
+ batch_size = len(prompt)
344
+ else:
345
+ batch_size = prompt_embeds.shape[0]
346
+
347
+ if prompt_embeds is None:
348
+ text_inputs = self.tokenizer(
349
+ prompt,
350
+ padding="max_length",
351
+ max_length=self.tokenizer.model_max_length,
352
+ truncation=True,
353
+ return_tensors="pt",
354
+ )
355
+ text_input_ids = text_inputs.input_ids
356
+ untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
357
+
358
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
359
+ text_input_ids, untruncated_ids
360
+ ):
361
+ removed_text = self.tokenizer.batch_decode(
362
+ untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]
363
+ )
364
+ logger.warning(
365
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
366
+ f" {self.tokenizer.model_max_length} tokens: {removed_text}"
367
+ )
368
+
369
+ if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:
370
+ attention_mask = text_inputs.attention_mask.to(device)
371
+ else:
372
+ attention_mask = None
373
+
374
+ prompt_embeds = self.text_encoder(
375
+ text_input_ids.to(device),
376
+ attention_mask=attention_mask,
377
+ )
378
+ prompt_embeds = prompt_embeds[0]
379
+
380
+ prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
381
+
382
+ bs_embed, seq_len, _ = prompt_embeds.shape
383
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
384
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
385
+ prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
386
+
387
+ # get unconditional embeddings for classifier free guidance
388
+ if do_classifier_free_guidance and negative_prompt_embeds is None:
389
+ uncond_tokens: List[str]
390
+ if negative_prompt is None:
391
+ uncond_tokens = [""] * batch_size
392
+ elif type(prompt) is not type(negative_prompt):
393
+ raise TypeError(
394
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
395
+ f" {type(prompt)}."
396
+ )
397
+ elif isinstance(negative_prompt, str):
398
+ uncond_tokens = [negative_prompt]
399
+ elif batch_size != len(negative_prompt):
400
+ raise ValueError(
401
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
402
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
403
+ " the batch size of `prompt`."
404
+ )
405
+ else:
406
+ uncond_tokens = negative_prompt
407
+
408
+ max_length = prompt_embeds.shape[1]
409
+ uncond_input = self.tokenizer(
410
+ uncond_tokens,
411
+ padding="max_length",
412
+ max_length=max_length,
413
+ truncation=True,
414
+ return_tensors="pt",
415
+ )
416
+
417
+ if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask:
418
+ attention_mask = uncond_input.attention_mask.to(device)
419
+ else:
420
+ attention_mask = None
421
+
422
+ negative_prompt_embeds = self.text_encoder(
423
+ uncond_input.input_ids.to(device),
424
+ attention_mask=attention_mask,
425
+ )
426
+ negative_prompt_embeds = negative_prompt_embeds[0]
427
+
428
+ if do_classifier_free_guidance:
429
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
430
+ seq_len = negative_prompt_embeds.shape[1]
431
+
432
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
433
+
434
+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
435
+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
436
+
437
+ # For classifier free guidance, we need to do two forward passes.
438
+ # Here we concatenate the unconditional and text embeddings into a single batch
439
+ # to avoid doing two forward passes
440
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
441
+
442
+ return prompt_embeds
443
+
444
+ def CLIP_preprocess(self, x):
445
+ dtype = x.dtype
446
+ # following openai's implementation
447
+ # TODO HF OpenAI CLIP preprocessing issue https://github.com/huggingface/transformers/issues/22505#issuecomment-1650170741
448
+ # follow openai preprocessing to keep exact same, input tensor [-1, 1], otherwise the preprocessing will be different, https://github.com/huggingface/transformers/pull/22608
449
+ if isinstance(x, torch.Tensor):
450
+ if x.min() < -1.0 or x.max() > 1.0:
451
+ raise ValueError("Expected input tensor to have values in the range [-1, 1]")
452
+ x = kornia.geometry.resize(x.to(torch.float32), (224, 224), interpolation='bicubic', align_corners=True, antialias=False).to(dtype=dtype)
453
+ x = (x + 1.) / 2.
454
+ # renormalize according to clip
455
+ x = kornia.enhance.normalize(x, torch.Tensor([0.48145466, 0.4578275, 0.40821073]),
456
+ torch.Tensor([0.26862954, 0.26130258, 0.27577711]))
457
+ return x
458
+
459
+ # from image_variation
460
+ def _encode_image(self, image, device, num_images_per_prompt, do_classifier_free_guidance):
461
+ dtype = next(self.image_encoder.parameters()).dtype
462
+ if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)):
463
+ raise ValueError(
464
+ f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"
465
+ )
466
+
467
+ if isinstance(image, torch.Tensor):
468
+ # Batch single image
469
+ if image.ndim == 3:
470
+ assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)"
471
+ image = image.unsqueeze(0)
472
+
473
+ assert image.ndim == 4, "Image must have 4 dimensions"
474
+
475
+ # Check image is in [-1, 1]
476
+ if image.min() < -1 or image.max() > 1:
477
+ raise ValueError("Image should be in [-1, 1] range")
478
+ else:
479
+ # preprocess image
480
+ if isinstance(image, (PIL.Image.Image, np.ndarray)):
481
+ image = [image]
482
+
483
+ if isinstance(image, list) and isinstance(image[0], PIL.Image.Image):
484
+ image = [np.array(i.convert("RGB"))[None, :] for i in image]
485
+ image = np.concatenate(image, axis=0)
486
+ elif isinstance(image, list) and isinstance(image[0], np.ndarray):
487
+ image = np.concatenate([i[None, :] for i in image], axis=0)
488
+
489
+ image = image.transpose(0, 3, 1, 2)
490
+ image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0
491
+
492
+ image = image.to(device=device, dtype=dtype)
493
+
494
+ image = self.CLIP_preprocess(image)
495
+ # if not isinstance(image, torch.Tensor):
496
+ # # 0-255
497
+ # print("Warning: image is processed by hf's preprocess, which is different from openai original's.")
498
+ # image = self.feature_extractor(images=image, return_tensors="pt").pixel_values
499
+ image_embeddings = self.image_encoder(image).image_embeds.to(dtype=dtype)
500
+ image_embeddings = image_embeddings.unsqueeze(1)
501
+
502
+ # duplicate image embeddings for each generation per prompt, using mps friendly method
503
+ bs_embed, seq_len, _ = image_embeddings.shape
504
+ image_embeddings = image_embeddings.repeat(1, num_images_per_prompt, 1)
505
+ image_embeddings = image_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)
506
+
507
+ if do_classifier_free_guidance:
508
+ negative_prompt_embeds = torch.zeros_like(image_embeddings)
509
+
510
+ # For classifier free guidance, we need to do two forward passes.
511
+ # Here we concatenate the unconditional and text embeddings into a single batch
512
+ # to avoid doing two forward passes
513
+ image_embeddings = torch.cat([negative_prompt_embeds, image_embeddings])
514
+
515
+ return image_embeddings
516
+
517
+ def _encode_pose(self, pose, device, num_images_per_prompt, do_classifier_free_guidance):
518
+ dtype = next(self.cc_projection.parameters()).dtype
519
+ if isinstance(pose, torch.Tensor):
520
+ pose_embeddings = pose.unsqueeze(1).to(device=device, dtype=dtype)
521
+ else:
522
+ if isinstance(pose[0], list):
523
+ pose = torch.Tensor(pose)
524
+ else:
525
+ pose = torch.Tensor([pose])
526
+ x, y, z = pose[:,0].unsqueeze(1), pose[:,1].unsqueeze(1), pose[:,2].unsqueeze(1)
527
+ pose_embeddings = torch.cat([torch.deg2rad(x),
528
+ torch.sin(torch.deg2rad(y)),
529
+ torch.cos(torch.deg2rad(y)),
530
+ z], dim=-1).unsqueeze(1).to(device=device, dtype=dtype) # B, 1, 4
531
+ # duplicate pose embeddings for each generation per prompt, using mps friendly method
532
+ bs_embed, seq_len, _ = pose_embeddings.shape
533
+ pose_embeddings = pose_embeddings.repeat(1, num_images_per_prompt, 1)
534
+ pose_embeddings = pose_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1)
535
+ if do_classifier_free_guidance:
536
+ negative_prompt_embeds = torch.zeros_like(pose_embeddings)
537
+
538
+ # For classifier free guidance, we need to do two forward passes.
539
+ # Here we concatenate the unconditional and text embeddings into a single batch
540
+ # to avoid doing two forward passes
541
+ pose_embeddings = torch.cat([negative_prompt_embeds, pose_embeddings])
542
+ return pose_embeddings
543
+
544
+ def _encode_image_with_pose(self, image, pose, device, num_images_per_prompt, do_classifier_free_guidance):
545
+ img_prompt_embeds = self._encode_image(image, device, num_images_per_prompt, False)
546
+ pose_prompt_embeds = self._encode_pose(pose, device, num_images_per_prompt, False)
547
+ prompt_embeds = torch.cat([img_prompt_embeds, pose_prompt_embeds], dim=-1)
548
+ prompt_embeds = self.cc_projection(prompt_embeds)
549
+ # prompt_embeds = img_prompt_embeds
550
+ # follow 0123, add negative prompt, after projection
551
+ if do_classifier_free_guidance:
552
+ negative_prompt = torch.zeros_like(prompt_embeds)
553
+ prompt_embeds = torch.cat([negative_prompt, prompt_embeds])
554
+ return prompt_embeds
555
+
556
+ def run_safety_checker(self, image, device, dtype):
557
+ if self.safety_checker is not None:
558
+ safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device)
559
+ image, has_nsfw_concept = self.safety_checker(
560
+ images=image, clip_input=safety_checker_input.pixel_values.to(dtype)
561
+ )
562
+ else:
563
+ has_nsfw_concept = None
564
+ return image, has_nsfw_concept
565
+
566
+ def decode_latents(self, latents):
567
+ latents = 1 / self.vae.config.scaling_factor * latents
568
+ image = self.vae.decode(latents).sample
569
+ image = (image / 2 + 0.5).clamp(0, 1)
570
+ # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
571
+ image = image.cpu().permute(0, 2, 3, 1).float().numpy()
572
+ return image
573
+
574
+ def prepare_extra_step_kwargs(self, generator, eta):
575
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
576
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
577
+ # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
578
+ # and should be between [0, 1]
579
+
580
+ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
581
+ extra_step_kwargs = {}
582
+ if accepts_eta:
583
+ extra_step_kwargs["eta"] = eta
584
+
585
+ # check if the scheduler accepts generator
586
+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
587
+ if accepts_generator:
588
+ extra_step_kwargs["generator"] = generator
589
+ return extra_step_kwargs
590
+
591
+ def check_inputs(self, image, height, width, callback_steps):
592
+ if (
593
+ not isinstance(image, torch.Tensor)
594
+ and not isinstance(image, PIL.Image.Image)
595
+ and not isinstance(image, list)
596
+ ):
597
+ raise ValueError(
598
+ "`image` has to be of type `torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is"
599
+ f" {type(image)}"
600
+ )
601
+
602
+ if height % 8 != 0 or width % 8 != 0:
603
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
604
+
605
+ if (callback_steps is None) or (
606
+ callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
607
+ ):
608
+ raise ValueError(
609
+ f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
610
+ f" {type(callback_steps)}."
611
+ )
612
+
613
+ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
614
+ shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)
615
+ if isinstance(generator, list) and len(generator) != batch_size:
616
+ raise ValueError(
617
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
618
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
619
+ )
620
+
621
+ if latents is None:
622
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
623
+ else:
624
+ latents = latents.to(device)
625
+
626
+ # scale the initial noise by the standard deviation required by the scheduler
627
+ latents = latents * self.scheduler.init_noise_sigma
628
+ return latents
629
+
630
+ def prepare_img_latents(self, image, batch_size, dtype, device, generator=None, do_classifier_free_guidance=False):
631
+ if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)):
632
+ raise ValueError(
633
+ f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}"
634
+ )
635
+
636
+ if isinstance(image, torch.Tensor):
637
+ # Batch single image
638
+ if image.ndim == 3:
639
+ assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)"
640
+ image = image.unsqueeze(0)
641
+
642
+ assert image.ndim == 4, "Image must have 4 dimensions"
643
+
644
+ # Check image is in [-1, 1]
645
+ if image.min() < -1 or image.max() > 1:
646
+ raise ValueError("Image should be in [-1, 1] range")
647
+ else:
648
+ # preprocess image
649
+ if isinstance(image, (PIL.Image.Image, np.ndarray)):
650
+ image = [image]
651
+
652
+ if isinstance(image, list) and isinstance(image[0], PIL.Image.Image):
653
+ image = [np.array(i.convert("RGB"))[None, :] for i in image]
654
+ image = np.concatenate(image, axis=0)
655
+ elif isinstance(image, list) and isinstance(image[0], np.ndarray):
656
+ image = np.concatenate([i[None, :] for i in image], axis=0)
657
+
658
+ image = image.transpose(0, 3, 1, 2)
659
+ image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0
660
+
661
+ image = image.to(device=device, dtype=dtype)
662
+
663
+ if isinstance(generator, list) and len(generator) != batch_size:
664
+ raise ValueError(
665
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
666
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
667
+ )
668
+
669
+ if isinstance(generator, list):
670
+ init_latents = [
671
+ self.vae.encode(image[i : i + 1]).latent_dist.mode(generator[i]) for i in range(batch_size) # sample
672
+ ]
673
+ init_latents = torch.cat(init_latents, dim=0)
674
+ else:
675
+ init_latents = self.vae.encode(image).latent_dist.mode()
676
+
677
+ # init_latents = self.vae.config.scaling_factor * init_latents # todo in original zero123's inference gradio_new.py, model.encode_first_stage() is not scaled by scaling_factor
678
+ if batch_size > init_latents.shape[0]:
679
+ # init_latents = init_latents.repeat(batch_size // init_latents.shape[0], 1, 1, 1)
680
+ num_images_per_prompt = batch_size // init_latents.shape[0]
681
+ # duplicate image latents for each generation per prompt, using mps friendly method
682
+ bs_embed, emb_c, emb_h, emb_w = init_latents.shape
683
+ init_latents = init_latents.unsqueeze(1)
684
+ init_latents = init_latents.repeat(1, num_images_per_prompt, 1, 1, 1)
685
+ init_latents = init_latents.view(bs_embed * num_images_per_prompt, emb_c, emb_h, emb_w)
686
+
687
+ # init_latents = torch.cat([init_latents]*2) if do_classifier_free_guidance else init_latents # follow zero123
688
+ init_latents = torch.cat([torch.zeros_like(init_latents), init_latents]) if do_classifier_free_guidance else init_latents
689
+
690
+ init_latents = init_latents.to(device=device, dtype=dtype)
691
+ return init_latents
692
+
693
+ # def load_cc_projection(self, pretrained_weights=None):
694
+ # self.cc_projection = torch.nn.Linear(772, 768)
695
+ # torch.nn.init.eye_(list(self.cc_projection.parameters())[0][:768, :768])
696
+ # torch.nn.init.zeros_(list(self.cc_projection.parameters())[1])
697
+ # if pretrained_weights is not None:
698
+ # self.cc_projection.load_state_dict(pretrained_weights)
699
+
700
+ @torch.no_grad()
701
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
702
+ def __call__(
703
+ self,
704
+ input_imgs: Union[torch.FloatTensor, PIL.Image.Image] = None,
705
+ prompt_imgs: Union[torch.FloatTensor, PIL.Image.Image] = None,
706
+ poses: Union[List[float], List[List[float]]] = None,
707
+ torch_dtype=torch.float32,
708
+ height: Optional[int] = None,
709
+ width: Optional[int] = None,
710
+ num_inference_steps: int = 50,
711
+ guidance_scale: float = 3.0,
712
+ negative_prompt: Optional[Union[str, List[str]]] = None,
713
+ num_images_per_prompt: Optional[int] = 1,
714
+ eta: float = 0.0,
715
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
716
+ latents: Optional[torch.FloatTensor] = None,
717
+ prompt_embeds: Optional[torch.FloatTensor] = None,
718
+ negative_prompt_embeds: Optional[torch.FloatTensor] = None,
719
+ output_type: Optional[str] = "pil",
720
+ return_dict: bool = True,
721
+ callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
722
+ callback_steps: int = 1,
723
+ cross_attention_kwargs: Optional[Dict[str, Any]] = None,
724
+ controlnet_conditioning_scale: float = 1.0,
725
+ ):
726
+ r"""
727
+ Function invoked when calling the pipeline for generation.
728
+
729
+ Args:
730
+ input_imgs (`PIL` or `List[PIL]`, *optional*):
731
+ The single input image for each 3D object
732
+ prompt_imgs (`PIL` or `List[PIL]`, *optional*):
733
+ Same as input_imgs, but will be used later as an image prompt condition, encoded by CLIP feature
734
+ height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
735
+ The height in pixels of the generated image.
736
+ width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
737
+ The width in pixels of the generated image.
738
+ num_inference_steps (`int`, *optional*, defaults to 50):
739
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
740
+ expense of slower inference.
741
+ guidance_scale (`float`, *optional*, defaults to 7.5):
742
+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
743
+ `guidance_scale` is defined as `w` of equation 2. of [Imagen
744
+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
745
+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
746
+ usually at the expense of lower image quality.
747
+ negative_prompt (`str` or `List[str]`, *optional*):
748
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
749
+ `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead.
750
+ Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`).
751
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
752
+ The number of images to generate per prompt.
753
+ eta (`float`, *optional*, defaults to 0.0):
754
+ Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to
755
+ [`schedulers.DDIMScheduler`], will be ignored for others.
756
+ generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
757
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
758
+ to make generation deterministic.
759
+ latents (`torch.FloatTensor`, *optional*):
760
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
761
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
762
+ tensor will ge generated by sampling using the supplied random `generator`.
763
+ prompt_embeds (`torch.FloatTensor`, *optional*):
764
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
765
+ provided, text embeddings will be generated from `prompt` input argument.
766
+ negative_prompt_embeds (`torch.FloatTensor`, *optional*):
767
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
768
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
769
+ argument.
770
+ output_type (`str`, *optional*, defaults to `"pil"`):
771
+ The output format of the generate image. Choose between
772
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
773
+ return_dict (`bool`, *optional*, defaults to `True`):
774
+ Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
775
+ plain tuple.
776
+ callback (`Callable`, *optional*):
777
+ A function that will be called every `callback_steps` steps during inference. The function will be
778
+ called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
779
+ callback_steps (`int`, *optional*, defaults to 1):
780
+ The frequency at which the `callback` function will be called. If not specified, the callback will be
781
+ called at every step.
782
+ cross_attention_kwargs (`dict`, *optional*):
783
+ A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under
784
+ `self.processor` in
785
+ [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py).
786
+
787
+ Examples:
788
+
789
+ Returns:
790
+ [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
791
+ [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple.
792
+ When returning a tuple, the first element is a list with the generated images, and the second element is a
793
+ list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work"
794
+ (nsfw) content, according to the `safety_checker`.
795
+ """
796
+ # 0. Default height and width to unet
797
+ height = height or self.unet.config.sample_size * self.vae_scale_factor
798
+ width = width or self.unet.config.sample_size * self.vae_scale_factor
799
+
800
+ # 1. Check inputs. Raise error if not correct
801
+ # input_image = hint_imgs
802
+ self.check_inputs(input_imgs, height, width, callback_steps)
803
+
804
+ # 2. Define call parameters
805
+ if isinstance(input_imgs, PIL.Image.Image):
806
+ batch_size = 1
807
+ elif isinstance(input_imgs, list):
808
+ batch_size = len(input_imgs)
809
+ else:
810
+ batch_size = input_imgs.shape[0]
811
+ device = self._execution_device
812
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
813
+ # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
814
+ # corresponds to doing no classifier free guidance.
815
+ do_classifier_free_guidance = guidance_scale > 1.0
816
+
817
+ # 3. Encode input image with pose as prompt
818
+ prompt_embeds = self._encode_image_with_pose(prompt_imgs, poses, device, num_images_per_prompt, do_classifier_free_guidance)
819
+
820
+ # 4. Prepare timesteps
821
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
822
+ timesteps = self.scheduler.timesteps
823
+
824
+ # 5. Prepare latent variables
825
+ latents = self.prepare_latents(
826
+ batch_size * num_images_per_prompt,
827
+ 4,
828
+ height,
829
+ width,
830
+ prompt_embeds.dtype,
831
+ device,
832
+ generator,
833
+ latents,
834
+ )
835
+
836
+ # 6. Prepare image latents
837
+ img_latents = self.prepare_img_latents(input_imgs,
838
+ batch_size * num_images_per_prompt,
839
+ prompt_embeds.dtype,
840
+ device,
841
+ generator,
842
+ do_classifier_free_guidance)
843
+
844
+ # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
845
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
846
+
847
+ # 7. Denoising loop
848
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
849
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
850
+ for i, t in enumerate(timesteps):
851
+ # expand the latents if we are doing classifier free guidance
852
+ latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
853
+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
854
+ latent_model_input = torch.cat([latent_model_input, img_latents], dim=1)
855
+
856
+ # predict the noise residual
857
+ noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=prompt_embeds).sample
858
+
859
+ # perform guidance
860
+ if do_classifier_free_guidance:
861
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
862
+ noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
863
+
864
+ # compute the previous noisy sample x_t -> x_t-1
865
+ # latents = self.scheduler.step(noise_pred.to(dtype=torch.float32), t, latents.to(dtype=torch.float32)).prev_sample.to(prompt_embeds.dtype)
866
+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
867
+
868
+ # call the callback, if provided
869
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
870
+ progress_bar.update()
871
+ if callback is not None and i % callback_steps == 0:
872
+ callback(i, t, latents)
873
+
874
+ # 8. Post-processing
875
+ has_nsfw_concept = None
876
+ if output_type == "latent":
877
+ image = latents
878
+ elif output_type == "pil":
879
+ # 8. Post-processing
880
+ image = self.decode_latents(latents)
881
+ # 10. Convert to PIL
882
+ image = self.numpy_to_pil(image)
883
+ else:
884
+ # 8. Post-processing
885
+ image = self.decode_latents(latents)
886
+
887
+ # Offload last model to CPU
888
+ if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None:
889
+ self.final_offload_hook.offload()
890
+
891
+ if not return_dict:
892
+ return (image, has_nsfw_concept)
893
+
894
+ return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)