city96

/

FLUX.1-schnell-gguf

like 304

Code :
from diffusers import FluxPipeline, FluxTransformer2DModel, GGUFQuantizationConfig
import torch
prompt = "a moonim dressed as a knight, riding a horse towards a medieval castle"

ckpt_path = "https://huggingface.co/city96/FLUX.1-schnell-gguf/blob/main/flux1-schnell-Q8_0.gguf"
transformer = FluxTransformer2DModel.from_single_file(
ckpt_path,
quantization_config=GGUFQuantizationConfig(compute_dtype=torch.bfloat16),
torch_dtype=torch.bfloat16,
)

pipe = FluxPipeline.from_pretrained(
"flux1schnell_model", # real flux 1 schnell model
transformer=transformer,
torch_dtype=torch.bfloat16,
)
pipe.enable_sequential_cpu_offload()
height, width = 1024, 1024
print("Starting Inference")
images = pipe(
prompt=prompt,
num_inference_steps=10,
guidance_scale=5.0,
height=height,
width=width,
generator=torch.Generator("cpu").manual_seed(42)
).images[0]
print("Output Ready")
images.save("gguf_image.png")

Error :
File "
/home/ubuntu/ghibli_poc/run_quant_flux.py", line 36, in
images = pipe(
^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/diffusers/pipelines/flux/pipeline_flux.py", line 889, in call
noise_pred = self.transformer(
^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/accelerate/hooks.py", line 175, in new_forward
output = module._old_forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/diffusers/models/transformers/transformer_flux.py", line 522, in forward
encoder_hidden_states, hidden_states = block(
^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/diffusers/models/transformers/transformer_flux.py", line 173, in forward
norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/diffusers/models/normalization.py", line 168, in forward
emb = self.linear(self.silu(emb))
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1532, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1541, in _call_impl
return forward_call(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/accelerate/hooks.py", line 175, in new_forward
output = module._old_forward(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/diffusers/quantizers/gguf/utils.py", line 451, in forward
weight = dequantize_gguf_tensor(self.weight)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ubuntu/ghibli_poc/venv/lib/python3.12/site-packages/diffusers/quantizers/gguf/utils.py", line 382, in dequantize_gguf_tensor
dequant_fn = dequantize_functions[quant_type]
~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^
KeyError: None