Delete MODELS/controlnet-canny-sdxl-1.0
Browse files- MODELS/controlnet-canny-sdxl-1.0/.gitattributes +0 -41
- MODELS/controlnet-canny-sdxl-1.0/README.md +0 -106
- MODELS/controlnet-canny-sdxl-1.0/config.json +0 -57
- MODELS/controlnet-canny-sdxl-1.0/diffusion_pytorch_model.fp16.safetensors +0 -3
- MODELS/controlnet-canny-sdxl-1.0/diffusion_pytorch_model.safetensors +0 -3
- MODELS/controlnet-canny-sdxl-1.0/out_bird.png +0 -3
- MODELS/controlnet-canny-sdxl-1.0/out_couple.png +0 -3
- MODELS/controlnet-canny-sdxl-1.0/out_hug_lab_7.png +0 -3
- MODELS/controlnet-canny-sdxl-1.0/out_room.png +0 -3
- MODELS/controlnet-canny-sdxl-1.0/out_tornado.png +0 -3
- MODELS/controlnet-canny-sdxl-1.0/out_women.png +0 -3
MODELS/controlnet-canny-sdxl-1.0/.gitattributes
DELETED
|
@@ -1,41 +0,0 @@
|
|
| 1 |
-
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
-
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
-
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
-
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
-
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
-
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
-
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
-
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
-
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
-
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
-
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
-
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
-
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
-
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
-
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
-
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
-
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
-
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
-
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
-
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
-
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
-
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
-
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
-
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
-
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
-
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
-
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
-
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
-
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
-
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
-
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
-
out_bird.png filter=lfs diff=lfs merge=lfs -text
|
| 37 |
-
out_couple.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
-
out_room.png filter=lfs diff=lfs merge=lfs -text
|
| 39 |
-
out_tornado.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
-
out_women.png filter=lfs diff=lfs merge=lfs -text
|
| 41 |
-
out_hug_lab_7.png filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
MODELS/controlnet-canny-sdxl-1.0/README.md
DELETED
|
@@ -1,106 +0,0 @@
|
|
| 1 |
-
---
|
| 2 |
-
license: openrail++
|
| 3 |
-
base_model: runwayml/stable-diffusion-v1-5
|
| 4 |
-
tags:
|
| 5 |
-
- stable-diffusion-xl
|
| 6 |
-
- stable-diffusion-xl-diffusers
|
| 7 |
-
- text-to-image
|
| 8 |
-
- diffusers
|
| 9 |
-
inference: false
|
| 10 |
-
---
|
| 11 |
-
|
| 12 |
-
# SDXL-controlnet: Canny
|
| 13 |
-
|
| 14 |
-
These are controlnet weights trained on stabilityai/stable-diffusion-xl-base-1.0 with canny conditioning. You can find some example images in the following.
|
| 15 |
-
|
| 16 |
-
prompt: a couple watching a romantic sunset, 4k photo
|
| 17 |
-

|
| 18 |
-
|
| 19 |
-
prompt: ultrarealistic shot of a furry blue bird
|
| 20 |
-

|
| 21 |
-
|
| 22 |
-
prompt: a woman, close up, detailed, beautiful, street photography, photorealistic, detailed, Kodak ektar 100, natural, candid shot
|
| 23 |
-

|
| 24 |
-
|
| 25 |
-
prompt: Cinematic, neoclassical table in the living room, cinematic, contour, lighting, highly detailed, winter, golden hour
|
| 26 |
-

|
| 27 |
-
|
| 28 |
-
prompt: a tornado hitting grass field, 1980's film grain. overcast, muted colors.
|
| 29 |
-

|
| 30 |
-
|
| 31 |
-
## Usage
|
| 32 |
-
|
| 33 |
-
Make sure to first install the libraries:
|
| 34 |
-
|
| 35 |
-
```bash
|
| 36 |
-
pip install accelerate transformers safetensors opencv-python diffusers
|
| 37 |
-
```
|
| 38 |
-
|
| 39 |
-
And then we're ready to go:
|
| 40 |
-
|
| 41 |
-
```python
|
| 42 |
-
from diffusers import ControlNetModel, StableDiffusionXLControlNetPipeline, AutoencoderKL
|
| 43 |
-
from diffusers.utils import load_image
|
| 44 |
-
from PIL import Image
|
| 45 |
-
import torch
|
| 46 |
-
import numpy as np
|
| 47 |
-
import cv2
|
| 48 |
-
|
| 49 |
-
prompt = "aerial view, a futuristic research complex in a bright foggy jungle, hard lighting"
|
| 50 |
-
negative_prompt = 'low quality, bad quality, sketches'
|
| 51 |
-
|
| 52 |
-
image = load_image("https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png")
|
| 53 |
-
|
| 54 |
-
controlnet_conditioning_scale = 0.5 # recommended for good generalization
|
| 55 |
-
|
| 56 |
-
controlnet = ControlNetModel.from_pretrained(
|
| 57 |
-
"diffusers/controlnet-canny-sdxl-1.0",
|
| 58 |
-
torch_dtype=torch.float16
|
| 59 |
-
)
|
| 60 |
-
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16)
|
| 61 |
-
pipe = StableDiffusionXLControlNetPipeline.from_pretrained(
|
| 62 |
-
"stabilityai/stable-diffusion-xl-base-1.0",
|
| 63 |
-
controlnet=controlnet,
|
| 64 |
-
vae=vae,
|
| 65 |
-
torch_dtype=torch.float16,
|
| 66 |
-
)
|
| 67 |
-
pipe.enable_model_cpu_offload()
|
| 68 |
-
|
| 69 |
-
image = np.array(image)
|
| 70 |
-
image = cv2.Canny(image, 100, 200)
|
| 71 |
-
image = image[:, :, None]
|
| 72 |
-
image = np.concatenate([image, image, image], axis=2)
|
| 73 |
-
image = Image.fromarray(image)
|
| 74 |
-
|
| 75 |
-
images = pipe(
|
| 76 |
-
prompt, negative_prompt=negative_prompt, image=image, controlnet_conditioning_scale=controlnet_conditioning_scale,
|
| 77 |
-
).images
|
| 78 |
-
|
| 79 |
-
images[0].save(f"hug_lab.png")
|
| 80 |
-
```
|
| 81 |
-
|
| 82 |
-

|
| 83 |
-
|
| 84 |
-
To more details, check out the official documentation of [`StableDiffusionXLControlNetPipeline`](https://huggingface.co/docs/diffusers/main/en/api/pipelines/controlnet_sdxl).
|
| 85 |
-
|
| 86 |
-
### Training
|
| 87 |
-
|
| 88 |
-
Our training script was built on top of the official training script that we provide [here](https://github.com/huggingface/diffusers/blob/main/examples/controlnet/README_sdxl.md).
|
| 89 |
-
|
| 90 |
-
#### Training data
|
| 91 |
-
This checkpoint was first trained for 20,000 steps on laion 6a resized to a max minimum dimension of 384.
|
| 92 |
-
It was then further trained for 20,000 steps on laion 6a resized to a max minimum dimension of 1024 and
|
| 93 |
-
then filtered to contain only minimum 1024 images. We found the further high resolution finetuning was
|
| 94 |
-
necessary for image quality.
|
| 95 |
-
|
| 96 |
-
#### Compute
|
| 97 |
-
one 8xA100 machine
|
| 98 |
-
|
| 99 |
-
#### Batch size
|
| 100 |
-
Data parallel with a single gpu batch size of 8 for a total batch size of 64.
|
| 101 |
-
|
| 102 |
-
#### Hyper Parameters
|
| 103 |
-
Constant learning rate of 1e-4 scaled by batch size for total learning rate of 64e-4
|
| 104 |
-
|
| 105 |
-
#### Mixed precision
|
| 106 |
-
fp16
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
MODELS/controlnet-canny-sdxl-1.0/config.json
DELETED
|
@@ -1,57 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"_class_name": "ControlNetModel",
|
| 3 |
-
"_diffusers_version": "0.20.0.dev0",
|
| 4 |
-
"_name_or_path": "../controlnet-1-0-canny/checkpoint-20000/controlnet",
|
| 5 |
-
"act_fn": "silu",
|
| 6 |
-
"addition_embed_type": "text_time",
|
| 7 |
-
"addition_embed_type_num_heads": 64,
|
| 8 |
-
"addition_time_embed_dim": 256,
|
| 9 |
-
"attention_head_dim": [
|
| 10 |
-
5,
|
| 11 |
-
10,
|
| 12 |
-
20
|
| 13 |
-
],
|
| 14 |
-
"block_out_channels": [
|
| 15 |
-
320,
|
| 16 |
-
640,
|
| 17 |
-
1280
|
| 18 |
-
],
|
| 19 |
-
"class_embed_type": null,
|
| 20 |
-
"conditioning_channels": 3,
|
| 21 |
-
"conditioning_embedding_out_channels": [
|
| 22 |
-
16,
|
| 23 |
-
32,
|
| 24 |
-
96,
|
| 25 |
-
256
|
| 26 |
-
],
|
| 27 |
-
"controlnet_conditioning_channel_order": "rgb",
|
| 28 |
-
"cross_attention_dim": 2048,
|
| 29 |
-
"down_block_types": [
|
| 30 |
-
"DownBlock2D",
|
| 31 |
-
"CrossAttnDownBlock2D",
|
| 32 |
-
"CrossAttnDownBlock2D"
|
| 33 |
-
],
|
| 34 |
-
"downsample_padding": 1,
|
| 35 |
-
"encoder_hid_dim": null,
|
| 36 |
-
"encoder_hid_dim_type": null,
|
| 37 |
-
"flip_sin_to_cos": true,
|
| 38 |
-
"freq_shift": 0,
|
| 39 |
-
"global_pool_conditions": false,
|
| 40 |
-
"in_channels": 4,
|
| 41 |
-
"layers_per_block": 2,
|
| 42 |
-
"mid_block_scale_factor": 1,
|
| 43 |
-
"norm_eps": 1e-05,
|
| 44 |
-
"norm_num_groups": 32,
|
| 45 |
-
"num_attention_heads": null,
|
| 46 |
-
"num_class_embeds": null,
|
| 47 |
-
"only_cross_attention": false,
|
| 48 |
-
"projection_class_embeddings_input_dim": 2816,
|
| 49 |
-
"resnet_time_scale_shift": "default",
|
| 50 |
-
"transformer_layers_per_block": [
|
| 51 |
-
1,
|
| 52 |
-
2,
|
| 53 |
-
10
|
| 54 |
-
],
|
| 55 |
-
"upcast_attention": null,
|
| 56 |
-
"use_linear_projection": true
|
| 57 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
MODELS/controlnet-canny-sdxl-1.0/diffusion_pytorch_model.fp16.safetensors
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:b2e7d3921058a442cc80430d1ec8847f42599c705e2451c95e77cf4dcf8d6c25
|
| 3 |
-
size 2502139136
|
|
|
|
|
|
|
|
|
|
|
|
MODELS/controlnet-canny-sdxl-1.0/diffusion_pytorch_model.safetensors
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:ea99040544a999f814fd854575a3aee069a005d026864c8d321b82576706a221
|
| 3 |
-
size 5004167864
|
|
|
|
|
|
|
|
|
|
|
|
MODELS/controlnet-canny-sdxl-1.0/out_bird.png
DELETED
Git LFS Details
|
MODELS/controlnet-canny-sdxl-1.0/out_couple.png
DELETED
Git LFS Details
|
MODELS/controlnet-canny-sdxl-1.0/out_hug_lab_7.png
DELETED
Git LFS Details
|
MODELS/controlnet-canny-sdxl-1.0/out_room.png
DELETED
Git LFS Details
|
MODELS/controlnet-canny-sdxl-1.0/out_tornado.png
DELETED
Git LFS Details
|
MODELS/controlnet-canny-sdxl-1.0/out_women.png
DELETED
Git LFS Details
|