Text-to-Image
Diffusers
Safetensors
StableDiffusionPipeline
clover-image
diffusion
stable-diffusion
knowledge-distillation
compact
small-model
local-inference
edge-inference
mobile-inference
core-ml
iphone
sd-1.4-class
Instructions to use neonforestmist/Clover-Image-Tiny with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Diffusers
How to use neonforestmist/Clover-Image-Tiny with Diffusers:
pip install -U diffusers transformers accelerate
import torch from diffusers import DiffusionPipeline # switch to "mps" for apple devices pipe = DiffusionPipeline.from_pretrained("neonforestmist/Clover-Image-Tiny", dtype=torch.bfloat16, device_map="cuda") prompt = "a glass of red wine" image = pipe(prompt).images[0] - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- Draw Things
- DiffusionBee
Commit ·
e042fb3
1
Parent(s): 4a6c93c
Release validated HQ inpainting pipelines
Browse files- coreml-tools/apple-no-mid-block.patch +54 -0
- coreml-tools/build_hq_manifest.py +100 -0
- coreml-tools/constraints.txt +5 -0
- coreml-tools/convert_stateful_multi_lora_unet.py +50 -13
- coreml-tools/validate_stateful_multi_lora.py +31 -3
- inpainting/evaluate_benchmark.py +40 -0
- modal_inpaint.py +12 -3
- modal_inpaint_benchmark.py +6 -1
- release/hq-v3/README-coreml.md +67 -0
- release/hq-v3/README-diffusers.md +121 -0
- release/hq-v3/manifest.json +103 -0
- release/hq-v4-int8/README-coreml.md +70 -0
- release/hq-v4-int8/manifest.json +106 -0
coreml-tools/apple-no-mid-block.patch
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
diff --git a/python_coreml_stable_diffusion/unet.py b/python_coreml_stable_diffusion/unet.py
|
| 2 |
+
index 666f146..519bc6f 100644
|
| 3 |
+
--- a/python_coreml_stable_diffusion/unet.py
|
| 4 |
+
+++ b/python_coreml_stable_diffusion/unet.py
|
| 5 |
+
@@ -916,19 +916,22 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin):
|
| 6 |
+
self.down_blocks.append(down_block)
|
| 7 |
+
|
| 8 |
+
# mid
|
| 9 |
+
- assert mid_block_type == "UNetMidBlock2DCrossAttn"
|
| 10 |
+
- self.mid_block = UNetMidBlock2DCrossAttn(
|
| 11 |
+
- in_channels=block_out_channels[-1],
|
| 12 |
+
- transformer_layers_per_block=transformer_layers_per_block[-1],
|
| 13 |
+
- temb_channels=time_embed_dim,
|
| 14 |
+
- resnet_eps=norm_eps,
|
| 15 |
+
- resnet_act_fn=act_fn,
|
| 16 |
+
- output_scale_factor=mid_block_scale_factor,
|
| 17 |
+
- resnet_time_scale_shift="default",
|
| 18 |
+
- cross_attention_dim=cross_attention_dim,
|
| 19 |
+
- attn_num_head_channels=attention_head_dim[i],
|
| 20 |
+
- resnet_groups=norm_num_groups,
|
| 21 |
+
- )
|
| 22 |
+
+ if mid_block_type is None:
|
| 23 |
+
+ self.mid_block = None
|
| 24 |
+
+ else:
|
| 25 |
+
+ assert mid_block_type == "UNetMidBlock2DCrossAttn"
|
| 26 |
+
+ self.mid_block = UNetMidBlock2DCrossAttn(
|
| 27 |
+
+ in_channels=block_out_channels[-1],
|
| 28 |
+
+ transformer_layers_per_block=transformer_layers_per_block[-1],
|
| 29 |
+
+ temb_channels=time_embed_dim,
|
| 30 |
+
+ resnet_eps=norm_eps,
|
| 31 |
+
+ resnet_act_fn=act_fn,
|
| 32 |
+
+ output_scale_factor=mid_block_scale_factor,
|
| 33 |
+
+ resnet_time_scale_shift="default",
|
| 34 |
+
+ cross_attention_dim=cross_attention_dim,
|
| 35 |
+
+ attn_num_head_channels=attention_head_dim[i],
|
| 36 |
+
+ resnet_groups=norm_num_groups,
|
| 37 |
+
+ )
|
| 38 |
+
|
| 39 |
+
# up
|
| 40 |
+
reversed_block_out_channels = list(reversed(block_out_channels))
|
| 41 |
+
@@ -1014,9 +1017,10 @@ class UNet2DConditionModel(ModelMixin, ConfigMixin):
|
| 42 |
+
down_block_res_samples = new_down_block_res_samples
|
| 43 |
+
|
| 44 |
+
# 4. mid
|
| 45 |
+
- sample = self.mid_block(sample,
|
| 46 |
+
- emb,
|
| 47 |
+
- encoder_hidden_states=encoder_hidden_states)
|
| 48 |
+
+ if self.mid_block is not None:
|
| 49 |
+
+ sample = self.mid_block(sample,
|
| 50 |
+
+ emb,
|
| 51 |
+
+ encoder_hidden_states=encoder_hidden_states)
|
| 52 |
+
|
| 53 |
+
if self.support_controlnet:
|
| 54 |
+
sample = sample + additional_residuals[-1]
|
coreml-tools/build_hq_manifest.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Build the pinned HQ inpainting download manifest from release artifacts."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import hashlib
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 12 |
+
COMPILED = (
|
| 13 |
+
ROOT
|
| 14 |
+
/ "coreml-inpaint-hq-quant"
|
| 15 |
+
/ "compiled"
|
| 16 |
+
/ "Stable_Diffusion_version_clover-image-tiny-inpaint-hq_unet_int8.mlmodelc"
|
| 17 |
+
)
|
| 18 |
+
SCHEMA = ROOT / "coreml-inpaint-hq-quant" / "adapter-schema.json"
|
| 19 |
+
OLD_MANIFEST = Path("/tmp/inpaint-coreml-manifest.json")
|
| 20 |
+
RELEASE_PATH = "hq-v4-int8"
|
| 21 |
+
OUTPUT = ROOT / "release" / RELEASE_PATH / "manifest.json"
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def record(path: Path, *, local_path: str, remote_path: str) -> dict[str, object]:
|
| 25 |
+
digest = hashlib.sha256()
|
| 26 |
+
with path.open("rb") as handle:
|
| 27 |
+
while chunk := handle.read(1024 * 1024):
|
| 28 |
+
digest.update(chunk)
|
| 29 |
+
return {
|
| 30 |
+
"path": local_path,
|
| 31 |
+
"remote_path": remote_path,
|
| 32 |
+
"size": path.stat().st_size,
|
| 33 |
+
"sha256": digest.hexdigest(),
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def main() -> None:
|
| 38 |
+
previous = json.loads(OLD_MANIFEST.read_text())
|
| 39 |
+
resources = [
|
| 40 |
+
item
|
| 41 |
+
for item in previous["resources"]
|
| 42 |
+
if item["path"].startswith("VAEEncoder.mlmodelc/")
|
| 43 |
+
]
|
| 44 |
+
for path in sorted(COMPILED.rglob("*")):
|
| 45 |
+
if path.is_file():
|
| 46 |
+
relative = path.relative_to(COMPILED).as_posix()
|
| 47 |
+
resources.append(
|
| 48 |
+
record(
|
| 49 |
+
path,
|
| 50 |
+
local_path=f"Unet.mlmodelc/{relative}",
|
| 51 |
+
remote_path=f"{RELEASE_PATH}/Unet.mlmodelc/{relative}",
|
| 52 |
+
)
|
| 53 |
+
)
|
| 54 |
+
resources.append(
|
| 55 |
+
record(
|
| 56 |
+
SCHEMA,
|
| 57 |
+
local_path="adapter-schema.json",
|
| 58 |
+
remote_path=f"{RELEASE_PATH}/adapter-schema.json",
|
| 59 |
+
)
|
| 60 |
+
)
|
| 61 |
+
manifest = {
|
| 62 |
+
"schema_version": 2,
|
| 63 |
+
"model": "neonforestmist/Clover-Image-Tiny-Inpaint",
|
| 64 |
+
"base_model": "neonforestmist/Clover-Image-Tiny",
|
| 65 |
+
"minimum_ios": "18.0",
|
| 66 |
+
"requires_base_model": True,
|
| 67 |
+
"resolution": [512, 512],
|
| 68 |
+
"max_adapter_count": 3,
|
| 69 |
+
"weight_compression": "per-channel symmetric int8",
|
| 70 |
+
"unet_input": {
|
| 71 |
+
"shape": [1, 9, 64, 64],
|
| 72 |
+
"channels": ["noisy_latent", "mask", "masked_image_latent"],
|
| 73 |
+
"mask_semantics": "white=regenerate, black=preserve",
|
| 74 |
+
},
|
| 75 |
+
"recommended_inference": {
|
| 76 |
+
"scheduler": "dpm_solver_multistep",
|
| 77 |
+
"steps": 20,
|
| 78 |
+
"guidance_scale": 6.0,
|
| 79 |
+
"mask_crop_padding": 96,
|
| 80 |
+
"composite_outside_mask": "exact source pixels",
|
| 81 |
+
},
|
| 82 |
+
"validation": {
|
| 83 |
+
"samples": 24,
|
| 84 |
+
"masked_clip_similarity": 0.27676651254296303,
|
| 85 |
+
"masked_target_mae": 0.2230850402265787,
|
| 86 |
+
"fp16_base_psnr_db": 78.55727510619732,
|
| 87 |
+
"fp16_three_style_psnr_db": 78.4737949968816,
|
| 88 |
+
"int8_base_psnr_db": 59.3153859192934,
|
| 89 |
+
"int8_three_style_psnr_db": 59.30338086829437,
|
| 90 |
+
},
|
| 91 |
+
"resources": sorted(resources, key=lambda item: item["path"]),
|
| 92 |
+
}
|
| 93 |
+
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
|
| 94 |
+
OUTPUT.write_text(json.dumps(manifest, indent=2) + "\n")
|
| 95 |
+
print(OUTPUT)
|
| 96 |
+
print(sum(int(item["size"]) for item in resources))
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
if __name__ == "__main__":
|
| 100 |
+
main()
|
coreml-tools/constraints.txt
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
coremltools==9.0
|
| 2 |
+
numpy==1.23.5
|
| 3 |
+
scikit-learn==1.5.1
|
| 4 |
+
scipy==1.11.4
|
| 5 |
+
torch==2.7.0
|
coreml-tools/convert_stateful_multi_lora_unet.py
CHANGED
|
@@ -18,6 +18,7 @@ import argparse
|
|
| 18 |
import gc
|
| 19 |
import json
|
| 20 |
import os
|
|
|
|
| 21 |
import time
|
| 22 |
from pathlib import Path
|
| 23 |
|
|
@@ -89,21 +90,49 @@ def inject_stateful_lora(
|
|
| 89 |
template: dict[str, torch.Tensor],
|
| 90 |
scale: float,
|
| 91 |
slot_count: int,
|
| 92 |
-
) ->
|
| 93 |
target_names = sorted(
|
| 94 |
{
|
| 95 |
key.removeprefix("unet.").split(".lora.")[0]
|
| 96 |
for key in template
|
| 97 |
}
|
| 98 |
)
|
| 99 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
parent_name, child_name = target_name.rsplit(".", 1)
|
| 101 |
parent = model.get_submodule(parent_name)
|
| 102 |
-
base = model.get_submodule(target_name)
|
| 103 |
-
if not isinstance(base, torch.nn.Conv2d):
|
| 104 |
-
raise TypeError(f"Expected Conv2d for {target_name}, found {type(base)}")
|
| 105 |
-
down = template[source_key(target_name, "lora_down")]
|
| 106 |
-
up = template[source_key(target_name, "lora_up")]
|
| 107 |
setattr(
|
| 108 |
parent,
|
| 109 |
child_name,
|
|
@@ -115,8 +144,12 @@ def inject_stateful_lora(
|
|
| 115 |
slot_count=slot_count,
|
| 116 |
),
|
| 117 |
)
|
|
|
|
| 118 |
if len(target_names) != 72:
|
| 119 |
raise RuntimeError(f"Expected 72 Clover LoRA targets, found {len(target_names)}")
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
|
| 122 |
def main() -> None:
|
|
@@ -134,15 +167,19 @@ def main() -> None:
|
|
| 134 |
raise ValueError("--max-adapter-count must be positive")
|
| 135 |
template = load_file(str(template_path))
|
| 136 |
original_class = coreml_unet.UNet2DConditionModel
|
|
|
|
| 137 |
|
| 138 |
class StatefulMultiLoRAUNet2DConditionModel(original_class):
|
| 139 |
def load_state_dict(self, state_dict, *args, **kwargs):
|
| 140 |
result = super().load_state_dict(state_dict, *args, **kwargs)
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
|
|
|
|
|
|
|
|
|
| 146 |
)
|
| 147 |
return result
|
| 148 |
|
|
@@ -210,7 +247,7 @@ def main() -> None:
|
|
| 210 |
)
|
| 211 |
for (torch_name, state_value), exported in zip(buffers, exported_states):
|
| 212 |
target_name, state_suffix = torch_name.rsplit(".", 1)
|
| 213 |
-
key = source_key(target_name, state_suffix)
|
| 214 |
source_value = template[key]
|
| 215 |
source_shape = list(source_value.shape)
|
| 216 |
source_shape.extend([1] * (state_value.ndim - source_value.ndim))
|
|
|
|
| 18 |
import gc
|
| 19 |
import json
|
| 20 |
import os
|
| 21 |
+
import re
|
| 22 |
import time
|
| 23 |
from pathlib import Path
|
| 24 |
|
|
|
|
| 90 |
template: dict[str, torch.Tensor],
|
| 91 |
scale: float,
|
| 92 |
slot_count: int,
|
| 93 |
+
) -> dict[str, str]:
|
| 94 |
target_names = sorted(
|
| 95 |
{
|
| 96 |
key.removeprefix("unet.").split(".lora.")[0]
|
| 97 |
for key in template
|
| 98 |
}
|
| 99 |
)
|
| 100 |
+
target_mapping: dict[str, str] = {}
|
| 101 |
+
for source_target_name in target_names:
|
| 102 |
+
down = template[source_key(source_target_name, "lora_down")]
|
| 103 |
+
up = template[source_key(source_target_name, "lora_up")]
|
| 104 |
+
candidates = [source_target_name]
|
| 105 |
+
up_match = re.match(r"up_blocks\.(\d+)\.(.+)", source_target_name)
|
| 106 |
+
if up_match:
|
| 107 |
+
# The HQ inpainting U-Net has an additional deepest UpBlock2D at
|
| 108 |
+
# index zero. Clover's three cross-attention up blocks therefore
|
| 109 |
+
# align with HQ indices one through three.
|
| 110 |
+
candidates.append(
|
| 111 |
+
f"up_blocks.{int(up_match.group(1)) + 1}.{up_match.group(2)}"
|
| 112 |
+
)
|
| 113 |
+
target_name = ""
|
| 114 |
+
base: torch.nn.Conv2d | None = None
|
| 115 |
+
for candidate in candidates:
|
| 116 |
+
try:
|
| 117 |
+
module = model.get_submodule(candidate)
|
| 118 |
+
except AttributeError:
|
| 119 |
+
continue
|
| 120 |
+
if not isinstance(module, torch.nn.Conv2d):
|
| 121 |
+
continue
|
| 122 |
+
if (
|
| 123 |
+
down.shape[1] == module.in_channels
|
| 124 |
+
and up.shape[0] == module.out_channels
|
| 125 |
+
):
|
| 126 |
+
target_name = candidate
|
| 127 |
+
base = module
|
| 128 |
+
break
|
| 129 |
+
if base is None:
|
| 130 |
+
raise AttributeError(
|
| 131 |
+
f"No channel-compatible HQ projection for {source_target_name}; "
|
| 132 |
+
f"tried {candidates}"
|
| 133 |
+
)
|
| 134 |
parent_name, child_name = target_name.rsplit(".", 1)
|
| 135 |
parent = model.get_submodule(parent_name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
setattr(
|
| 137 |
parent,
|
| 138 |
child_name,
|
|
|
|
| 144 |
slot_count=slot_count,
|
| 145 |
),
|
| 146 |
)
|
| 147 |
+
target_mapping[target_name] = source_target_name
|
| 148 |
if len(target_names) != 72:
|
| 149 |
raise RuntimeError(f"Expected 72 Clover LoRA targets, found {len(target_names)}")
|
| 150 |
+
if len(target_mapping) != len(target_names):
|
| 151 |
+
raise RuntimeError("LoRA target mapping contains duplicate HQ targets")
|
| 152 |
+
return target_mapping
|
| 153 |
|
| 154 |
|
| 155 |
def main() -> None:
|
|
|
|
| 167 |
raise ValueError("--max-adapter-count must be positive")
|
| 168 |
template = load_file(str(template_path))
|
| 169 |
original_class = coreml_unet.UNet2DConditionModel
|
| 170 |
+
target_source_names: dict[str, str] = {}
|
| 171 |
|
| 172 |
class StatefulMultiLoRAUNet2DConditionModel(original_class):
|
| 173 |
def load_state_dict(self, state_dict, *args, **kwargs):
|
| 174 |
result = super().load_state_dict(state_dict, *args, **kwargs)
|
| 175 |
+
target_source_names.clear()
|
| 176 |
+
target_source_names.update(
|
| 177 |
+
inject_stateful_lora(
|
| 178 |
+
self,
|
| 179 |
+
template=template,
|
| 180 |
+
scale=wrapper_args.lora_scale,
|
| 181 |
+
slot_count=wrapper_args.max_adapter_count,
|
| 182 |
+
)
|
| 183 |
)
|
| 184 |
return result
|
| 185 |
|
|
|
|
| 247 |
)
|
| 248 |
for (torch_name, state_value), exported in zip(buffers, exported_states):
|
| 249 |
target_name, state_suffix = torch_name.rsplit(".", 1)
|
| 250 |
+
key = source_key(target_source_names[target_name], state_suffix)
|
| 251 |
source_value = template[key]
|
| 252 |
source_shape = list(source_value.shape)
|
| 253 |
source_shape.extend([1] * (state_value.ndim - source_value.ndim))
|
coreml-tools/validate_stateful_multi_lora.py
CHANGED
|
@@ -6,6 +6,7 @@ from __future__ import annotations
|
|
| 6 |
import argparse
|
| 7 |
import json
|
| 8 |
import math
|
|
|
|
| 9 |
from pathlib import Path
|
| 10 |
|
| 11 |
import coremltools as ct
|
|
@@ -54,10 +55,34 @@ def inject_loras(model, adapters, scales):
|
|
| 54 |
}
|
| 55 |
)
|
| 56 |
for target_name in target_names:
|
| 57 |
-
parent_name, child_name = target_name.rsplit(".", 1)
|
| 58 |
-
parent = model.get_submodule(parent_name)
|
| 59 |
-
base = model.get_submodule(target_name)
|
| 60 |
prefix = f"unet.{target_name}.lora"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
components = [
|
| 62 |
(
|
| 63 |
state[f"{prefix}.down.weight"],
|
|
@@ -116,6 +141,9 @@ def main():
|
|
| 116 |
pipeline = DiffusionPipeline.from_pretrained(
|
| 117 |
args.model_version,
|
| 118 |
local_files_only=True,
|
|
|
|
|
|
|
|
|
|
| 119 |
)
|
| 120 |
reference = coreml_unet.UNet2DConditionModel(**pipeline.unet.config).eval()
|
| 121 |
reference.load_state_dict(pipeline.unet.state_dict())
|
|
|
|
| 6 |
import argparse
|
| 7 |
import json
|
| 8 |
import math
|
| 9 |
+
import re
|
| 10 |
from pathlib import Path
|
| 11 |
|
| 12 |
import coremltools as ct
|
|
|
|
| 55 |
}
|
| 56 |
)
|
| 57 |
for target_name in target_names:
|
|
|
|
|
|
|
|
|
|
| 58 |
prefix = f"unet.{target_name}.lora"
|
| 59 |
+
down_shape = states[0][f"{prefix}.down.weight"].shape
|
| 60 |
+
up_shape = states[0][f"{prefix}.up.weight"].shape
|
| 61 |
+
candidates = [target_name]
|
| 62 |
+
up_match = re.match(r"up_blocks\.(\d+)\.(.+)", target_name)
|
| 63 |
+
if up_match:
|
| 64 |
+
candidates.append(
|
| 65 |
+
f"up_blocks.{int(up_match.group(1)) + 1}.{up_match.group(2)}"
|
| 66 |
+
)
|
| 67 |
+
resolved_name = ""
|
| 68 |
+
base = None
|
| 69 |
+
for candidate in candidates:
|
| 70 |
+
try:
|
| 71 |
+
module = model.get_submodule(candidate)
|
| 72 |
+
except AttributeError:
|
| 73 |
+
continue
|
| 74 |
+
if (
|
| 75 |
+
isinstance(module, torch.nn.Conv2d)
|
| 76 |
+
and down_shape[1] == module.in_channels
|
| 77 |
+
and up_shape[0] == module.out_channels
|
| 78 |
+
):
|
| 79 |
+
resolved_name = candidate
|
| 80 |
+
base = module
|
| 81 |
+
break
|
| 82 |
+
if base is None:
|
| 83 |
+
raise AttributeError(f"No compatible projection for {target_name}")
|
| 84 |
+
parent_name, child_name = resolved_name.rsplit(".", 1)
|
| 85 |
+
parent = model.get_submodule(parent_name)
|
| 86 |
components = [
|
| 87 |
(
|
| 88 |
state[f"{prefix}.down.weight"],
|
|
|
|
| 141 |
pipeline = DiffusionPipeline.from_pretrained(
|
| 142 |
args.model_version,
|
| 143 |
local_files_only=True,
|
| 144 |
+
torch_dtype=torch.float16,
|
| 145 |
+
variant="fp16",
|
| 146 |
+
use_safetensors=True,
|
| 147 |
)
|
| 148 |
reference = coreml_unet.UNet2DConditionModel(**pipeline.unet.config).eval()
|
| 149 |
reference.load_state_dict(pipeline.unet.state_dict())
|
inpainting/evaluate_benchmark.py
CHANGED
|
@@ -45,6 +45,46 @@ def _model_specs(values: list[str]) -> list[tuple[str, str]]:
|
|
| 45 |
|
| 46 |
|
| 47 |
def _load_pipeline(model_path: str, device: torch.device):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
pipeline_path, separator, unet_state_path = model_path.partition("::")
|
| 49 |
pipeline = StableDiffusionInpaintPipeline.from_pretrained(
|
| 50 |
pipeline_path,
|
|
|
|
| 45 |
|
| 46 |
|
| 47 |
def _load_pipeline(model_path: str, device: torch.device):
|
| 48 |
+
if model_path.startswith("blend@"):
|
| 49 |
+
_, alpha_text, candidate_path = model_path.split("@", 2)
|
| 50 |
+
alpha = float(alpha_text)
|
| 51 |
+
if not 0.0 <= alpha <= 1.0:
|
| 52 |
+
raise ValueError("blend alpha must be in [0, 1]")
|
| 53 |
+
pipeline = StableDiffusionInpaintPipeline.from_pretrained(
|
| 54 |
+
"neonforestmist/Clover-Image-Tiny-Inpaint",
|
| 55 |
+
torch_dtype=torch.float16,
|
| 56 |
+
safety_checker=None,
|
| 57 |
+
requires_safety_checker=False,
|
| 58 |
+
)
|
| 59 |
+
candidate = load_file(candidate_path)
|
| 60 |
+
blended = pipeline.unet.state_dict()
|
| 61 |
+
for name, value in blended.items():
|
| 62 |
+
if name not in candidate:
|
| 63 |
+
raise KeyError(f"Candidate is missing U-Net tensor {name}")
|
| 64 |
+
update = candidate[name].to(dtype=value.dtype)
|
| 65 |
+
blended[name] = value.lerp(update, alpha)
|
| 66 |
+
pipeline.unet.load_state_dict(blended, strict=True)
|
| 67 |
+
return pipeline.to(device)
|
| 68 |
+
if model_path.startswith("clover-hybrid::"):
|
| 69 |
+
teacher_path = model_path.removeprefix("clover-hybrid::")
|
| 70 |
+
pipeline = StableDiffusionInpaintPipeline.from_pretrained(
|
| 71 |
+
teacher_path,
|
| 72 |
+
torch_dtype=torch.float16,
|
| 73 |
+
safety_checker=None,
|
| 74 |
+
requires_safety_checker=False,
|
| 75 |
+
)
|
| 76 |
+
clover = StableDiffusionInpaintPipeline.from_pretrained(
|
| 77 |
+
"neonforestmist/Clover-Image-Tiny-Inpaint",
|
| 78 |
+
torch_dtype=torch.float16,
|
| 79 |
+
safety_checker=None,
|
| 80 |
+
requires_safety_checker=False,
|
| 81 |
+
)
|
| 82 |
+
pipeline.tokenizer = clover.tokenizer
|
| 83 |
+
pipeline.text_encoder = clover.text_encoder
|
| 84 |
+
pipeline.vae = clover.vae
|
| 85 |
+
pipeline.scheduler = clover.scheduler
|
| 86 |
+
del clover
|
| 87 |
+
return pipeline.to(device)
|
| 88 |
pipeline_path, separator, unet_state_path = model_path.partition("::")
|
| 89 |
pipeline = StableDiffusionInpaintPipeline.from_pretrained(
|
| 90 |
pipeline_path,
|
modal_inpaint.py
CHANGED
|
@@ -48,7 +48,12 @@ app = modal.App(
|
|
| 48 |
)
|
| 49 |
|
| 50 |
|
| 51 |
-
@app.function(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
def train(
|
| 53 |
*,
|
| 54 |
base_model: str = "neonforestmist/Clover-Image-Tiny",
|
|
@@ -72,6 +77,7 @@ def train(
|
|
| 72 |
lr_warmup_steps: int | None = None,
|
| 73 |
train_batch_size: int = 1,
|
| 74 |
gradient_accumulation_steps: int = 4,
|
|
|
|
| 75 |
lora_rank: int = 0,
|
| 76 |
teacher_loss_weight: float = 0.75,
|
| 77 |
ground_truth_loss_weight: float = 0.25,
|
|
@@ -136,7 +142,6 @@ def train(
|
|
| 136 |
str(gradient_accumulation_steps),
|
| 137 |
"--train_batch_size",
|
| 138 |
str(train_batch_size),
|
| 139 |
-
"--gradient_checkpointing",
|
| 140 |
"--mixed_precision",
|
| 141 |
"bf16",
|
| 142 |
"--random_flip",
|
|
@@ -165,6 +170,8 @@ def train(
|
|
| 165 |
"--output_dir",
|
| 166 |
str(output_dir),
|
| 167 |
]
|
|
|
|
|
|
|
| 168 |
if initial_inpaint_revision:
|
| 169 |
command.extend(["--initial_inpaint_revision", initial_inpaint_revision])
|
| 170 |
if lora_rank > 0:
|
|
@@ -221,6 +228,7 @@ def main(
|
|
| 221 |
lr_warmup_steps: int | None = None,
|
| 222 |
train_batch_size: int = 1,
|
| 223 |
gradient_accumulation_steps: int = 4,
|
|
|
|
| 224 |
object_fraction: float = 0.70,
|
| 225 |
lora_rank: int = 0,
|
| 226 |
teacher_loss_weight: float = 0.75,
|
|
@@ -230,7 +238,7 @@ def main(
|
|
| 230 |
boundary_loss_weight: float = 2.0,
|
| 231 |
resume: bool = False,
|
| 232 |
) -> None:
|
| 233 |
-
"""Launch a smoke test or
|
| 234 |
|
| 235 |
if smoke:
|
| 236 |
steps = min(steps, 4)
|
|
@@ -250,6 +258,7 @@ def main(
|
|
| 250 |
lr_warmup_steps=lr_warmup_steps,
|
| 251 |
train_batch_size=train_batch_size,
|
| 252 |
gradient_accumulation_steps=gradient_accumulation_steps,
|
|
|
|
| 253 |
object_fraction=object_fraction,
|
| 254 |
lora_rank=lora_rank,
|
| 255 |
teacher_loss_weight=teacher_loss_weight,
|
|
|
|
| 48 |
)
|
| 49 |
|
| 50 |
|
| 51 |
+
@app.function(
|
| 52 |
+
gpu=os.environ.get("CLOVER_MODAL_GPU", "A10"),
|
| 53 |
+
timeout=24 * 60 * 60,
|
| 54 |
+
cpu=8,
|
| 55 |
+
memory=32768,
|
| 56 |
+
)
|
| 57 |
def train(
|
| 58 |
*,
|
| 59 |
base_model: str = "neonforestmist/Clover-Image-Tiny",
|
|
|
|
| 77 |
lr_warmup_steps: int | None = None,
|
| 78 |
train_batch_size: int = 1,
|
| 79 |
gradient_accumulation_steps: int = 4,
|
| 80 |
+
gradient_checkpointing: bool = True,
|
| 81 |
lora_rank: int = 0,
|
| 82 |
teacher_loss_weight: float = 0.75,
|
| 83 |
ground_truth_loss_weight: float = 0.25,
|
|
|
|
| 142 |
str(gradient_accumulation_steps),
|
| 143 |
"--train_batch_size",
|
| 144 |
str(train_batch_size),
|
|
|
|
| 145 |
"--mixed_precision",
|
| 146 |
"bf16",
|
| 147 |
"--random_flip",
|
|
|
|
| 170 |
"--output_dir",
|
| 171 |
str(output_dir),
|
| 172 |
]
|
| 173 |
+
if gradient_checkpointing:
|
| 174 |
+
command.append("--gradient_checkpointing")
|
| 175 |
if initial_inpaint_revision:
|
| 176 |
command.extend(["--initial_inpaint_revision", initial_inpaint_revision])
|
| 177 |
if lora_rank > 0:
|
|
|
|
| 228 |
lr_warmup_steps: int | None = None,
|
| 229 |
train_batch_size: int = 1,
|
| 230 |
gradient_accumulation_steps: int = 4,
|
| 231 |
+
gradient_checkpointing: bool = True,
|
| 232 |
object_fraction: float = 0.70,
|
| 233 |
lora_rank: int = 0,
|
| 234 |
teacher_loss_weight: float = 0.75,
|
|
|
|
| 238 |
boundary_loss_weight: float = 2.0,
|
| 239 |
resume: bool = False,
|
| 240 |
) -> None:
|
| 241 |
+
"""Launch a smoke test or a bounded v3 recipe on the selected Modal GPU."""
|
| 242 |
|
| 243 |
if smoke:
|
| 244 |
steps = min(steps, 4)
|
|
|
|
| 258 |
lr_warmup_steps=lr_warmup_steps,
|
| 259 |
train_batch_size=train_batch_size,
|
| 260 |
gradient_accumulation_steps=gradient_accumulation_steps,
|
| 261 |
+
gradient_checkpointing=gradient_checkpointing,
|
| 262 |
object_fraction=object_fraction,
|
| 263 |
lora_rank=lora_rank,
|
| 264 |
teacher_loss_weight=teacher_loss_weight,
|
modal_inpaint_benchmark.py
CHANGED
|
@@ -43,7 +43,12 @@ app = modal.App(
|
|
| 43 |
)
|
| 44 |
|
| 45 |
|
| 46 |
-
@app.function(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
def benchmark(*, models: list[str], output_name: str, samples: int = 24) -> str:
|
| 48 |
output_dir = OUTPUT_ROOT / output_name
|
| 49 |
if output_dir.exists():
|
|
|
|
| 43 |
)
|
| 44 |
|
| 45 |
|
| 46 |
+
@app.function(
|
| 47 |
+
gpu=os.environ.get("CLOVER_MODAL_GPU", "A10"),
|
| 48 |
+
timeout=4 * 60 * 60,
|
| 49 |
+
cpu=8,
|
| 50 |
+
memory=32768,
|
| 51 |
+
)
|
| 52 |
def benchmark(*, models: list[str], output_name: str, samples: int = 24) -> str:
|
| 53 |
output_dir = OUTPUT_ROOT / output_name
|
| 54 |
if output_dir.exists():
|
release/hq-v3/README-coreml.md
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: coreml
|
| 3 |
+
pipeline_tag: image-to-image
|
| 4 |
+
base_model: neonforestmist/Clover-Image-Tiny-Inpaint
|
| 5 |
+
license: creativeml-openrail-m
|
| 6 |
+
tags:
|
| 7 |
+
- coreml
|
| 8 |
+
- ios
|
| 9 |
+
- inpainting
|
| 10 |
+
- lora
|
| 11 |
+
- stable-diffusion
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+
# Clover Image Tiny Inpaint HQ — Core ML
|
| 15 |
+
|
| 16 |
+
Core ML resources for the high-quality Clover Image Tiny inpainting pipeline.
|
| 17 |
+
The iOS 18 batch-one U-Net accepts `[1, 9, 64, 64]`, runs classifier-free
|
| 18 |
+
guidance as two serial passes, and exposes 144 mutable Core ML states for exact
|
| 19 |
+
runtime composition of up to three Clover styles.
|
| 20 |
+
|
| 21 |
+
The HQ U-Net is under `hq-v3/Unet.mlmodelc`. Shared Clover resources remain at
|
| 22 |
+
the repository root so app upgrades only download files whose checksums
|
| 23 |
+
changed. `hq-v3/adapter-schema.json` maps each small `.safetensors` style into
|
| 24 |
+
one of three independent state slots.
|
| 25 |
+
|
| 26 |
+
## Validation
|
| 27 |
+
|
| 28 |
+
PyTorch/Core ML parity on deterministic inputs:
|
| 29 |
+
|
| 30 |
+
| Configuration | PSNR |
|
| 31 |
+
|---|---:|
|
| 32 |
+
| Base HQ inpainting U-Net | **78.56 dB** |
|
| 33 |
+
| Monet 0.70 + Pointillism 0.45 + Watercolor Anime 1.10 | **78.47 dB** |
|
| 34 |
+
|
| 35 |
+
The release gate is 35 dB. The three-style result validates the exact
|
| 36 |
+
block-concatenated LoRA sum rather than a UI-only approximation.
|
| 37 |
+
|
| 38 |
+
Held-out 24-case inpainting quality relative to the previous release:
|
| 39 |
+
|
| 40 |
+
| Metric | Previous | HQ |
|
| 41 |
+
|---|---:|---:|
|
| 42 |
+
| Masked prompt CLIP similarity | 0.2642 | **0.2768** |
|
| 43 |
+
| Masked target MAE | 0.2510 | **0.2231** |
|
| 44 |
+
| Changed pixels outside the mask | 0 | **0** |
|
| 45 |
+
|
| 46 |
+
## Runtime contract
|
| 47 |
+
|
| 48 |
+
- Minimum OS: iOS 18
|
| 49 |
+
- Resolution: 512×512
|
| 50 |
+
- U-Net input: noisy latent (4) + mask (1) + masked image latent (4)
|
| 51 |
+
- Mask: white regenerates, black preserves
|
| 52 |
+
- Maximum simultaneous styles: 3
|
| 53 |
+
- Safety checker: not constructed by the Clover iOS pipeline
|
| 54 |
+
|
| 55 |
+
## Citation
|
| 56 |
+
|
| 57 |
+
```bibtex
|
| 58 |
+
@software{lozadaperez2026cloverimagetinyinpaintcoreml,
|
| 59 |
+
author = {Lukas Lozada Perez},
|
| 60 |
+
title = {Clover Image Tiny Inpaint HQ Core ML},
|
| 61 |
+
year = {2026},
|
| 62 |
+
url = {https://huggingface.co/neonforestmist/Clover-Image-Tiny-Inpaint-CoreML}
|
| 63 |
+
}
|
| 64 |
+
```
|
| 65 |
+
|
| 66 |
+
Designed and developed independently by Lukas Lozada Perez. Open weights under
|
| 67 |
+
the CreativeML Open RAIL-M license; inference runs completely on device.
|
release/hq-v3/README-diffusers.md
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: diffusers
|
| 3 |
+
pipeline_tag: image-to-image
|
| 4 |
+
base_model:
|
| 5 |
+
- neonforestmist/Clover-Image-Tiny
|
| 6 |
+
- stable-diffusion-v1-5/stable-diffusion-inpainting
|
| 7 |
+
license: creativeml-openrail-m
|
| 8 |
+
tags:
|
| 9 |
+
- clover-image
|
| 10 |
+
- inpainting
|
| 11 |
+
- stable-diffusion
|
| 12 |
+
- coreml
|
| 13 |
+
---
|
| 14 |
+
|
| 15 |
+
# Clover Image Tiny Inpaint HQ
|
| 16 |
+
|
| 17 |
+
Clover Image Tiny Inpaint HQ is the high-quality, context-aware inpainting
|
| 18 |
+
pipeline for Clover. It combines the complete Stable Diffusion 1.5 inpainting
|
| 19 |
+
U-Net with Clover Image Tiny's tokenizer, text encoder, VAE, and scheduler.
|
| 20 |
+
This preserves Clover compatibility while replacing the compact inpainting
|
| 21 |
+
denoiser that frequently produced blurry or unrecognizable masked objects.
|
| 22 |
+
|
| 23 |
+
The pipeline uses the standard nine-channel inpainting contract:
|
| 24 |
+
|
| 25 |
+
```text
|
| 26 |
+
[noisy latent (4), mask (1), masked-image latent (4)]
|
| 27 |
+
```
|
| 28 |
+
|
| 29 |
+
## Diffusers example
|
| 30 |
+
|
| 31 |
+
```python
|
| 32 |
+
import torch
|
| 33 |
+
from diffusers import AutoPipelineForInpainting, DPMSolverMultistepScheduler
|
| 34 |
+
from diffusers.utils import load_image
|
| 35 |
+
|
| 36 |
+
pipe = AutoPipelineForInpainting.from_pretrained(
|
| 37 |
+
"neonforestmist/Clover-Image-Tiny-Inpaint",
|
| 38 |
+
torch_dtype=torch.float16,
|
| 39 |
+
).to("cuda")
|
| 40 |
+
pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
|
| 41 |
+
|
| 42 |
+
image = pipe(
|
| 43 |
+
prompt="a glossy red enamel kettle resting naturally on the countertop",
|
| 44 |
+
negative_prompt="blurry, distorted, low detail",
|
| 45 |
+
image=load_image("source.png"),
|
| 46 |
+
mask_image=load_image("mask.png"),
|
| 47 |
+
num_inference_steps=20,
|
| 48 |
+
guidance_scale=6.0,
|
| 49 |
+
padding_mask_crop=96,
|
| 50 |
+
).images[0]
|
| 51 |
+
image.save("clover-inpaint.png")
|
| 52 |
+
```
|
| 53 |
+
|
| 54 |
+
Recommended interactive defaults are DPM-Solver++, 20 steps, CFG 6.0, and a
|
| 55 |
+
96-pixel context crop. Composite the generated result through the exact binary
|
| 56 |
+
mask when unchanged source pixels must remain byte-for-byte untouched.
|
| 57 |
+
|
| 58 |
+
## Quality gate
|
| 59 |
+
|
| 60 |
+
The release was evaluated on 24 deterministic, held-out, human-rated
|
| 61 |
+
InpaintCOCO edits. Every output was also reviewed in three visual contact
|
| 62 |
+
sheets before release.
|
| 63 |
+
|
| 64 |
+
| Metric | Previous Clover inpaint | HQ release | SD 1.5 inpaint teacher |
|
| 65 |
+
|---|---:|---:|---:|
|
| 66 |
+
| Masked prompt CLIP similarity (higher) | 0.2642 | **0.2768** | 0.2820 |
|
| 67 |
+
| Masked target MAE (lower) | 0.2510 | **0.2231** | 0.2156 |
|
| 68 |
+
| Changed pixels outside the mask | 0 | **0** | 0 |
|
| 69 |
+
|
| 70 |
+
The HQ release improves prompt alignment by 4.8% and reduces masked target
|
| 71 |
+
error by 11.1% relative to the previous Clover inpainting release. The visual
|
| 72 |
+
gate showed recognizable buses, dogs, trains, furniture, signs, and
|
| 73 |
+
scene-consistent lighting where the compact candidates often collapsed into
|
| 74 |
+
amorphous fills.
|
| 75 |
+
|
| 76 |
+
## Selection provenance
|
| 77 |
+
|
| 78 |
+
The release process compared the existing checkpoint, a 30,000-step full-U-Net
|
| 79 |
+
distillation run, two fused context-LoRA refinements, partial weight blends,
|
| 80 |
+
the full Stable Diffusion inpainting reference, and this Clover-component
|
| 81 |
+
hybrid. The 30,000-step and context-LoRA candidates were rejected because they
|
| 82 |
+
did not beat the existing release across both visual and quantitative gates.
|
| 83 |
+
The published HQ architecture was the only Clover-compatible candidate that
|
| 84 |
+
materially improved both prompt alignment and reconstruction.
|
| 85 |
+
|
| 86 |
+
- Inpainting U-Net revision:
|
| 87 |
+
`stable-diffusion-v1-5/stable-diffusion-inpainting@8a4288a76071f7280aedbdb3253bdb9e9d5d84bb`
|
| 88 |
+
- Clover components: `neonforestmist/Clover-Image-Tiny`
|
| 89 |
+
- Evaluation dataset: `phiyodr/InpaintCOCO@1ffac84be2dfc5ad9afccad868522fad64457435`
|
| 90 |
+
- Selection platform: Modal H100
|
| 91 |
+
- Evaluation seed: `20260813`
|
| 92 |
+
|
| 93 |
+
## Core ML and style mixing
|
| 94 |
+
|
| 95 |
+
The companion iOS resources are published at
|
| 96 |
+
[`neonforestmist/Clover-Image-Tiny-Inpaint-CoreML`](https://huggingface.co/neonforestmist/Clover-Image-Tiny-Inpaint-CoreML).
|
| 97 |
+
Its batch-one stateful U-Net supports up to three Clover styles simultaneously
|
| 98 |
+
with independent strengths. The style tensors remain separate downloads and
|
| 99 |
+
are composed exactly at runtime; they are not fused into three full 1.6 GB
|
| 100 |
+
models.
|
| 101 |
+
|
| 102 |
+
## Limitations
|
| 103 |
+
|
| 104 |
+
Small text, hands, faces, exact logos, and masks below latent resolution can
|
| 105 |
+
still fail. Output quality depends on the source, mask, prompt, scheduler,
|
| 106 |
+
guidance, seed, and step count. This release inherits the limitations and
|
| 107 |
+
license obligations of Clover Image Tiny and Stable Diffusion 1.5 inpainting.
|
| 108 |
+
|
| 109 |
+
## Citation
|
| 110 |
+
|
| 111 |
+
```bibtex
|
| 112 |
+
@software{lozadaperez2026cloverimagetinyinpaint,
|
| 113 |
+
author = {Lukas Lozada Perez},
|
| 114 |
+
title = {Clover Image Tiny Inpaint HQ: Local Context-Aware Image Inpainting},
|
| 115 |
+
year = {2026},
|
| 116 |
+
url = {https://huggingface.co/neonforestmist/Clover-Image-Tiny-Inpaint}
|
| 117 |
+
}
|
| 118 |
+
```
|
| 119 |
+
|
| 120 |
+
Designed and developed independently by Lukas Lozada Perez. Open weights under
|
| 121 |
+
the CreativeML Open RAIL-M license; complete local inference is supported.
|
release/hq-v3/manifest.json
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 2,
|
| 3 |
+
"model": "neonforestmist/Clover-Image-Tiny-Inpaint",
|
| 4 |
+
"base_model": "neonforestmist/Clover-Image-Tiny",
|
| 5 |
+
"minimum_ios": "18.0",
|
| 6 |
+
"requires_base_model": true,
|
| 7 |
+
"resolution": [
|
| 8 |
+
512,
|
| 9 |
+
512
|
| 10 |
+
],
|
| 11 |
+
"max_adapter_count": 3,
|
| 12 |
+
"unet_input": {
|
| 13 |
+
"shape": [
|
| 14 |
+
1,
|
| 15 |
+
9,
|
| 16 |
+
64,
|
| 17 |
+
64
|
| 18 |
+
],
|
| 19 |
+
"channels": [
|
| 20 |
+
"noisy_latent",
|
| 21 |
+
"mask",
|
| 22 |
+
"masked_image_latent"
|
| 23 |
+
],
|
| 24 |
+
"mask_semantics": "white=regenerate, black=preserve"
|
| 25 |
+
},
|
| 26 |
+
"recommended_inference": {
|
| 27 |
+
"scheduler": "dpm_solver_multistep",
|
| 28 |
+
"steps": 20,
|
| 29 |
+
"guidance_scale": 6.0,
|
| 30 |
+
"mask_crop_padding": 96,
|
| 31 |
+
"composite_outside_mask": "exact source pixels"
|
| 32 |
+
},
|
| 33 |
+
"validation": {
|
| 34 |
+
"samples": 24,
|
| 35 |
+
"masked_clip_similarity": 0.27676651254296303,
|
| 36 |
+
"masked_target_mae": 0.2230850402265787,
|
| 37 |
+
"base_psnr_db": 78.55727510619732,
|
| 38 |
+
"three_style_psnr_db": 78.4737949968816
|
| 39 |
+
},
|
| 40 |
+
"resources": [
|
| 41 |
+
{
|
| 42 |
+
"path": "Unet.mlmodelc/analytics/coremldata.bin",
|
| 43 |
+
"remote_path": "hq-v3/Unet.mlmodelc/analytics/coremldata.bin",
|
| 44 |
+
"size": 243,
|
| 45 |
+
"sha256": "0776280c592bd31e478eecd0cac5918984d48e629d1ffa5a9d6432a698448c9e"
|
| 46 |
+
},
|
| 47 |
+
{
|
| 48 |
+
"path": "Unet.mlmodelc/coremldata.bin",
|
| 49 |
+
"remote_path": "hq-v3/Unet.mlmodelc/coremldata.bin",
|
| 50 |
+
"size": 14009,
|
| 51 |
+
"sha256": "4cb24ed826e3e090138e99e3eb334e706d6b8bf733771fe034355268456e6834"
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"path": "Unet.mlmodelc/metadata.json",
|
| 55 |
+
"remote_path": "hq-v3/Unet.mlmodelc/metadata.json",
|
| 56 |
+
"size": 50802,
|
| 57 |
+
"sha256": "54a5d730ce8790f028faacfe44cf9b555fb3109153cb5ffd8016ff5deb0375d8"
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"path": "Unet.mlmodelc/model.mil",
|
| 61 |
+
"remote_path": "hq-v3/Unet.mlmodelc/model.mil",
|
| 62 |
+
"size": 1206218,
|
| 63 |
+
"sha256": "2f7238a5825ae16c9dc054536d9a5dd7cf0963af2423979b4f84ae9121c6606a"
|
| 64 |
+
},
|
| 65 |
+
{
|
| 66 |
+
"path": "Unet.mlmodelc/weights/weight.bin",
|
| 67 |
+
"remote_path": "hq-v3/Unet.mlmodelc/weights/weight.bin",
|
| 68 |
+
"size": 1719146496,
|
| 69 |
+
"sha256": "0f6a88bcd1ae4f36c076d46d404373c2f5c44bf85975ed2737b23643b9040826"
|
| 70 |
+
},
|
| 71 |
+
{
|
| 72 |
+
"path": "VAEEncoder.mlmodelc/analytics/coremldata.bin",
|
| 73 |
+
"size": 243,
|
| 74 |
+
"sha256": "ba122c04ae4c8d28c8b8805049e22fb7dfeac3eada7bde68c2366a14e712d1d6"
|
| 75 |
+
},
|
| 76 |
+
{
|
| 77 |
+
"path": "VAEEncoder.mlmodelc/coremldata.bin",
|
| 78 |
+
"size": 883,
|
| 79 |
+
"sha256": "fe1d98a943015c7ac3d29ef2d573b37610e62c97e5724d2f85c555960832629d"
|
| 80 |
+
},
|
| 81 |
+
{
|
| 82 |
+
"path": "VAEEncoder.mlmodelc/metadata.json",
|
| 83 |
+
"size": 2674,
|
| 84 |
+
"sha256": "f18ce0aabcf1ad6695c8d7bf418dd30bdac1d031ef453fe1f2b655487dff019d"
|
| 85 |
+
},
|
| 86 |
+
{
|
| 87 |
+
"path": "VAEEncoder.mlmodelc/model.mil",
|
| 88 |
+
"size": 132453,
|
| 89 |
+
"sha256": "52c0b22663abb7890a76932f8c8b2467908bfc5b6cdef92e4a2cbcde9a7d07a4"
|
| 90 |
+
},
|
| 91 |
+
{
|
| 92 |
+
"path": "VAEEncoder.mlmodelc/weights/weight.bin",
|
| 93 |
+
"size": 68338112,
|
| 94 |
+
"sha256": "71bf826e5ee1b455462aff0b34e3a469f5f58955ae715708962b1d81db92dc64"
|
| 95 |
+
},
|
| 96 |
+
{
|
| 97 |
+
"path": "adapter-schema.json",
|
| 98 |
+
"remote_path": "hq-v3/adapter-schema.json",
|
| 99 |
+
"size": 55773,
|
| 100 |
+
"sha256": "b356bae467a4e60cb26151c5b2cad2d2dc298b057f44b71479096036ca53c1d9"
|
| 101 |
+
}
|
| 102 |
+
]
|
| 103 |
+
}
|
release/hq-v4-int8/README-coreml.md
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
library_name: coreml
|
| 3 |
+
pipeline_tag: image-to-image
|
| 4 |
+
base_model: neonforestmist/Clover-Image-Tiny-Inpaint
|
| 5 |
+
license: creativeml-openrail-m
|
| 6 |
+
tags:
|
| 7 |
+
- coreml
|
| 8 |
+
- ios
|
| 9 |
+
- inpainting
|
| 10 |
+
- lora
|
| 11 |
+
- stable-diffusion
|
| 12 |
+
---
|
| 13 |
+
|
| 14 |
+
# Clover Image Tiny Inpaint HQ — Core ML
|
| 15 |
+
|
| 16 |
+
Core ML resources for the high-quality Clover Image Tiny inpainting pipeline.
|
| 17 |
+
The iOS 18 batch-one U-Net accepts `[1, 9, 64, 64]`, runs classifier-free
|
| 18 |
+
guidance as two serial passes, and exposes 144 mutable Core ML states for exact
|
| 19 |
+
runtime composition of up to three Clover styles.
|
| 20 |
+
|
| 21 |
+
The production U-Net is under `hq-v4-int8/Unet.mlmodelc`. Its immutable
|
| 22 |
+
weights use per-channel symmetric int8 compression, reducing the complete
|
| 23 |
+
inpainting add-on from about 1.79 GB to **931 MB** while leaving mutable LoRA
|
| 24 |
+
state in FP16. Shared tokenizer, text encoder, and VAE decoder resources are
|
| 25 |
+
hard-linked from the required main Clover installation instead of downloaded
|
| 26 |
+
again. `hq-v4-int8/adapter-schema.json` maps each small `.safetensors` style
|
| 27 |
+
into one of three independent state slots.
|
| 28 |
+
|
| 29 |
+
## Validation
|
| 30 |
+
|
| 31 |
+
PyTorch/Core ML parity on deterministic inputs:
|
| 32 |
+
|
| 33 |
+
| Configuration | FP16 | Int8 production |
|
| 34 |
+
|---|---:|---:|
|
| 35 |
+
| Base HQ inpainting U-Net | 78.56 dB | **59.32 dB** |
|
| 36 |
+
| Monet 0.70 + Pointillism 0.45 + Watercolor Anime 1.10 | 78.47 dB | **59.30 dB** |
|
| 37 |
+
|
| 38 |
+
The release gate is 35 dB. The three-style result validates the exact
|
| 39 |
+
block-concatenated LoRA sum rather than a UI-only approximation.
|
| 40 |
+
|
| 41 |
+
Held-out 24-case inpainting quality relative to the previous Diffusers release:
|
| 42 |
+
|
| 43 |
+
| Metric | Previous | HQ |
|
| 44 |
+
|---|---:|---:|
|
| 45 |
+
| Masked prompt CLIP similarity | 0.2642 | **0.2768** |
|
| 46 |
+
| Masked target MAE | 0.2510 | **0.2231** |
|
| 47 |
+
| Changed pixels outside the mask | 0 | **0** |
|
| 48 |
+
|
| 49 |
+
## Runtime contract
|
| 50 |
+
|
| 51 |
+
- Minimum OS: iOS 18
|
| 52 |
+
- Resolution: 512×512
|
| 53 |
+
- U-Net input: noisy latent (4) + mask (1) + masked image latent (4)
|
| 54 |
+
- Mask: white regenerates, black preserves
|
| 55 |
+
- Maximum simultaneous styles: 3
|
| 56 |
+
- Safety checker: not constructed by the Clover iOS pipeline
|
| 57 |
+
|
| 58 |
+
## Citation
|
| 59 |
+
|
| 60 |
+
```bibtex
|
| 61 |
+
@software{lozadaperez2026cloverimagetinyinpaintcoreml,
|
| 62 |
+
author = {Lukas Lozada Perez},
|
| 63 |
+
title = {Clover Image Tiny Inpaint HQ Core ML},
|
| 64 |
+
year = {2026},
|
| 65 |
+
url = {https://huggingface.co/neonforestmist/Clover-Image-Tiny-Inpaint-CoreML}
|
| 66 |
+
}
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
Designed and developed independently by Lukas Lozada Perez. Open weights under
|
| 70 |
+
the CreativeML Open RAIL-M license; inference runs completely on device.
|
release/hq-v4-int8/manifest.json
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"schema_version": 2,
|
| 3 |
+
"model": "neonforestmist/Clover-Image-Tiny-Inpaint",
|
| 4 |
+
"base_model": "neonforestmist/Clover-Image-Tiny",
|
| 5 |
+
"minimum_ios": "18.0",
|
| 6 |
+
"requires_base_model": true,
|
| 7 |
+
"resolution": [
|
| 8 |
+
512,
|
| 9 |
+
512
|
| 10 |
+
],
|
| 11 |
+
"max_adapter_count": 3,
|
| 12 |
+
"weight_compression": "per-channel symmetric int8",
|
| 13 |
+
"unet_input": {
|
| 14 |
+
"shape": [
|
| 15 |
+
1,
|
| 16 |
+
9,
|
| 17 |
+
64,
|
| 18 |
+
64
|
| 19 |
+
],
|
| 20 |
+
"channels": [
|
| 21 |
+
"noisy_latent",
|
| 22 |
+
"mask",
|
| 23 |
+
"masked_image_latent"
|
| 24 |
+
],
|
| 25 |
+
"mask_semantics": "white=regenerate, black=preserve"
|
| 26 |
+
},
|
| 27 |
+
"recommended_inference": {
|
| 28 |
+
"scheduler": "dpm_solver_multistep",
|
| 29 |
+
"steps": 20,
|
| 30 |
+
"guidance_scale": 6.0,
|
| 31 |
+
"mask_crop_padding": 96,
|
| 32 |
+
"composite_outside_mask": "exact source pixels"
|
| 33 |
+
},
|
| 34 |
+
"validation": {
|
| 35 |
+
"samples": 24,
|
| 36 |
+
"masked_clip_similarity": 0.27676651254296303,
|
| 37 |
+
"masked_target_mae": 0.2230850402265787,
|
| 38 |
+
"fp16_base_psnr_db": 78.55727510619732,
|
| 39 |
+
"fp16_three_style_psnr_db": 78.4737949968816,
|
| 40 |
+
"int8_base_psnr_db": 59.3153859192934,
|
| 41 |
+
"int8_three_style_psnr_db": 59.30338086829437
|
| 42 |
+
},
|
| 43 |
+
"resources": [
|
| 44 |
+
{
|
| 45 |
+
"path": "Unet.mlmodelc/analytics/coremldata.bin",
|
| 46 |
+
"remote_path": "hq-v4-int8/Unet.mlmodelc/analytics/coremldata.bin",
|
| 47 |
+
"size": 243,
|
| 48 |
+
"sha256": "fe4fd138e2daa944e7ce8aabdcc7fd371bea3f0fbd3add90c4534af74645c641"
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"path": "Unet.mlmodelc/coremldata.bin",
|
| 52 |
+
"remote_path": "hq-v4-int8/Unet.mlmodelc/coremldata.bin",
|
| 53 |
+
"size": 14009,
|
| 54 |
+
"sha256": "cb75c645b04342e3c99a57577b88a74a006bde0bbf7ae758174868083717856c"
|
| 55 |
+
},
|
| 56 |
+
{
|
| 57 |
+
"path": "Unet.mlmodelc/metadata.json",
|
| 58 |
+
"remote_path": "hq-v4-int8/Unet.mlmodelc/metadata.json",
|
| 59 |
+
"size": 50871,
|
| 60 |
+
"sha256": "6485785471f213843cdb8133e07a0e76d1e6dc9667e59047ac4a67c75db899ea"
|
| 61 |
+
},
|
| 62 |
+
{
|
| 63 |
+
"path": "Unet.mlmodelc/model.mil",
|
| 64 |
+
"remote_path": "hq-v4-int8/Unet.mlmodelc/model.mil",
|
| 65 |
+
"size": 1256410,
|
| 66 |
+
"sha256": "f47ba899b3b7b47dcea95109d4ab8f59372b6cf24df25f83b094dd770d60afb2"
|
| 67 |
+
},
|
| 68 |
+
{
|
| 69 |
+
"path": "Unet.mlmodelc/weights/weight.bin",
|
| 70 |
+
"remote_path": "hq-v4-int8/Unet.mlmodelc/weights/weight.bin",
|
| 71 |
+
"size": 860709760,
|
| 72 |
+
"sha256": "142f9f803d75ea2379ca3793c7d03b5da9eb29ac59fbae1a6dc5fe38b864e8a5"
|
| 73 |
+
},
|
| 74 |
+
{
|
| 75 |
+
"path": "VAEEncoder.mlmodelc/analytics/coremldata.bin",
|
| 76 |
+
"size": 243,
|
| 77 |
+
"sha256": "ba122c04ae4c8d28c8b8805049e22fb7dfeac3eada7bde68c2366a14e712d1d6"
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
"path": "VAEEncoder.mlmodelc/coremldata.bin",
|
| 81 |
+
"size": 883,
|
| 82 |
+
"sha256": "fe1d98a943015c7ac3d29ef2d573b37610e62c97e5724d2f85c555960832629d"
|
| 83 |
+
},
|
| 84 |
+
{
|
| 85 |
+
"path": "VAEEncoder.mlmodelc/metadata.json",
|
| 86 |
+
"size": 2674,
|
| 87 |
+
"sha256": "f18ce0aabcf1ad6695c8d7bf418dd30bdac1d031ef453fe1f2b655487dff019d"
|
| 88 |
+
},
|
| 89 |
+
{
|
| 90 |
+
"path": "VAEEncoder.mlmodelc/model.mil",
|
| 91 |
+
"size": 132453,
|
| 92 |
+
"sha256": "52c0b22663abb7890a76932f8c8b2467908bfc5b6cdef92e4a2cbcde9a7d07a4"
|
| 93 |
+
},
|
| 94 |
+
{
|
| 95 |
+
"path": "VAEEncoder.mlmodelc/weights/weight.bin",
|
| 96 |
+
"size": 68338112,
|
| 97 |
+
"sha256": "71bf826e5ee1b455462aff0b34e3a469f5f58955ae715708962b1d81db92dc64"
|
| 98 |
+
},
|
| 99 |
+
{
|
| 100 |
+
"path": "adapter-schema.json",
|
| 101 |
+
"remote_path": "hq-v4-int8/adapter-schema.json",
|
| 102 |
+
"size": 55773,
|
| 103 |
+
"sha256": "b356bae467a4e60cb26151c5b2cad2d2dc298b057f44b71479096036ca53c1d9"
|
| 104 |
+
}
|
| 105 |
+
]
|
| 106 |
+
}
|