start for chimera

app.py

@@ -0,0 +1,121 @@
+import torch
+from transformers import TextIteratorStreamer, AutoProcessor, LlavaForConditionalGeneration
+from diffusers import DiffusionPipeline
+import gradio as gr
+import numpy as np
+import accelerate
+import spaces
+from PIL import Image
+import threading
+
+DESCRIPTION = '''
+<div>
+<h1 style="text-align: center;">Krypton 🕋</h1>
+<p>This uses an Open Source model from <a href="https://huggingface.co/xtuner/llava-llama-3-8b-v1_1-transformers"><b>xtuner/llava-llama-3-8b-v1_1-transformers</b></a></p>
+</div>
+'''
+# Llava Installed
+llava_model = LlavaForConditionalGeneration.from_pretrained(
+    "xtuner/llava-llama-3-8b-v1_1-transformers",
+    torch_dtype=torch.float16,
+    low_cpu_mem_usage=True,
+    )
+
+llava_model.to("cuda:0")
+
+processor = AutoProcessor.from_pretrained("xtuner/llava-llama-3-8b-v1_1-transformers")
+
+llava_model.generation_config.eos_token_id=128009
+
+# Stable Diffusor Installed
+base = DiffusionPipeline.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0",
+    torch_dtype=torch.float16,
+    variant="fp16",
+    use_safetensors=True,
+)
+base.to('cuda')
+
+refiner = DiffusionPipeline.from_pretrained(
+    "stabilityai/stable-diffusion-xl-base-1.0",
+    text_encoder_2=base.text_encoder_2,
+    vae=base.vae,
+    torch_dtype=torch.float16,
+    use_safetensors=True,
+    variant="fp16",
+)
+refiner.to('cuda')
+
+# All Installed. Let's instance them in the function
+
+def chimera(message, history):
+    """
+    Receives input from gradio from the prompt but also
+    if any images were passed that i also placed for formatting
+    for PIL and with the prompt both are passed to proper generation,
+    depending on the request from prompt, that prompt output will return here.
+    """
+    print(f"Message:\n{message}\nType:\n{type.message}")
+    if message["files"]:
+        if type(message["files"][-1]) == dict:
+            image_path = message["files"][-1]["path"]
+        else:
+            image_path = message["files"][-1]
+    else:
+        # If no image was uploaded than look for past ones
+        for hist in history:
+            if type(hist[0]) == tuple:
+                image_path = hist[0][0] # item inside items for history
+
+    prompt = f"<|start_header_id|>user<|end_header_id|>\n\n<image>\n{message['text']}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
+
+    if image_path is None:
+        image = base(
+            prompt=prompt,
+            num_inference_steps=40,
+            denoising_end=0.8,
+            output_type="latent",
+        ).images
+        image = refiner(
+            prompt=prompt,
+            num_inference_steps=40,
+            denoising_start=0.8,
+            image=image
+        ).images[0]
+        return image
+
+    else:
+
+        # Time to instance the llava
+        image = Image.open(image_path)
+        inputs = processor(prompt, image, return_tensors='pt').to(0, torch.float16)
+        streamer = TextIteratorStreamer(processor, **{"skip_special_tokens": False, "skip_prompt": True})
+        generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=1024, do_sample=False)
+
+        thread = threading.Thread(target=llava_model.generate, kwargs=generation_kwargs)
+        thread.start()
+
+        buffer = ""
+        for new_text in streamer:
+            # find <|eot_id|> and remove it from the new_text
+            if "<|eot_id|>" in new_text:
+                new_text = new_text.split("<|eot_id|>")[0]
+            buffer += new_text
+            generated_text_no_prompt = buffer
+            yield generated_text_no_prompt
+
+    
+chatbot=gr.Chatbot(height=600, label="Chimera AI")
+chat_input = gr.MultimodalTextbox(interactive=True, file_types=["images"], placeholder="Enter your question or upload an image.", show_label=False)
+with gr.Blocks(fill_height=True) as demo:
+    gr.Markdown(DESCRIPTION)
+    gr.ChatInterface(
+        fn=chimera,
+        chatbot=chatbot,
+        fill_height=True,
+        multimodal=True,
+        textbox=chat_input,
+    )
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,6 @@
+torch
+transformers
+gradio
+numpy
+accelerate
+diffusers