allow additional reference images, resize image before use

app.py

@@ -5,13 +5,15 @@ import torch
 from threading import Thread
 import requests
 import json
+import io
+from PIL import Image
 import os
 import base64
 from openai import OpenAI
 
 default_img = None
 default_base_url = "https://openrouter.ai/api/v1"
-default_api_model = "google/gemma-3-27b-it:free"
+default_api_model = "google/gemma-3-27b-it"
 
 model_id = "google/gemma-3-4b-it"
 
@@ -32,10 +34,13 @@ lang_store = {
         "confirm": "Confirm",
         "default_description": "",
         "additional_description": "Character description (optional)",
-        "title": "<h1>Chat with a character via reference sheet!</h1>",
+        "more_imgs": "More reference images of the character (optional)",
+        "title": "<h1>Chat with a character via reference sheet!</h1>>",
+        "powered_by_gemma": "<p>Powered by <a href='https://blog.google/technology/developers/gemma-3/'>Gemma 3</a></p",
         "upload": "Upload the reference sheet of your character here",
-        "prompt": "You are the character in the image. Start without confirmation.",
+        "prompt": "You are the character in the image. Do not include list in response unless requested. Do not mention the reference images. Start without confirmation.",
         "additional_info_prompt": "Additional info: ",
+        "additional_reference_images_prompt": "Additional reference images of the character:",
         "description": "Description",
         "more_options": "More Options",
         "method": "Method",
@@ -49,10 +54,13 @@ lang_store = {
         "confirm": "确认",
         "default_description": "",
         "additional_description": "角色描述（可选）",
+        "more_imgs": "更多角色参考图（可选，可上传多张）",
         "title": "<h1>与设定图中的角色聊天！</h1>",
+        "powered_by_gemma": "<p>由 <a href='https://blog.google/technology/developers/gemma-3/'>Gemma 3</a> 驱动</p>",
         "upload": "在这里上传角色设定图",
-        "prompt": "你的身份是图中的角色，使用中文。无需确认。",
+        "prompt": "你的身份是图中的角色，使用中文。除非对方要求，否则不在回复中使用列表。不在回复中提及参考图。无需确认。",
         "additional_info_prompt": "补充信息：",
+        "additional_reference_images_prompt": "该角色的更多参考图：",
         "description": "角色描述",
         "more_options": "更多选项",
         "method": "方法",
@@ -64,17 +72,31 @@ lang_store = {
     },
 }
 
-def get_init_prompt(img, description):
+def encode_img(filepath, thumbnail=(896, 896)):
+    more_img = Image.open(filepath)
+    more_img = more_img.convert('RGB')
+    more_img.thumbnail(thumbnail)
+    buffer = io.BytesIO()
+    more_img.save(buffer, "JPEG", quality=60)
+    encoded_img = "data:image/jpeg;base64," + base64.b64encode(buffer.getvalue()).decode("utf-8")
+    return encoded_img
+
+def get_init_prompt(img, description, more_imgs):
     prompt = _("prompt")
     if description != "":
-        prompt += _("additional_info_prompt") + description
+        prompt += "\n" + _("additional_info_prompt") + description
+    if more_imgs is None:
+        more_imgs = []
+    if len(more_imgs) > 0:
+        prompt += "\n" + _("additional_reference_images_prompt")
+    content = [
+        {"type": "image", "url": encode_img(img)},
+        {"type": "text", "text": prompt}
+    ] + [{"type": "image", "url": encode_img(filepath)} for filepath in more_imgs]
     return [
         {
             "role": "user",
-            "content": [
-                {"type": "image", "url": img},
-                {"type": "text", "text": prompt}
-            ]
+            "content": content
         }
     ]
 
@@ -101,9 +123,7 @@ def generate(history, engine, base_url, api_model, api_key):
             for item_i in item['content']:
                 if item_i['type'] == 'image':
                     item_i['type'] = 'image_url'
-                    with open(item_i['url'], "rb") as image_file:
-                        data = base64.b64encode(image_file.read()).decode("utf-8")
-                    item_i['image_url'] = {'url': 'data:image/jpeg;base64,' + data}
+                    item_i['image_url'] = {'url': item_i['url']}
                     del item_i['url']
         if base_url == default_base_url and api_model == default_api_model and api_key == "":
             api_key = os.environ['OPENROUTER_TOKEN']
@@ -122,8 +142,8 @@ def generate(history, engine, base_url, api_model, api_key):
                 yield collected_text
 
 
-def prefill_chatbot(img, description, engine, base_url, api_model, api_key):
-    history = get_init_prompt(img, description)
+def prefill_chatbot(img, description, more_imgs, engine, base_url, api_model, api_key):
+    history = get_init_prompt(img, description, more_imgs)
 
     ret = [{'role': 'assistant', 'content': ""}]
     for generated_text in generate(history, engine, base_url, api_model, api_key):
@@ -131,9 +151,9 @@ def prefill_chatbot(img, description, engine, base_url, api_model, api_key):
         yield ret
 
 
-def response(message, history: list, img, description, engine, base_url, api_model, api_key):
+def response(message, history: list, img, description, more_imgs, engine, base_url, api_model, api_key):
     history = [{"role": item["role"], "content": [{"type": "text", "text": item["content"]}]} for item in history]
-    history = get_init_prompt(img, description) + history
+    history = get_init_prompt(img, description, more_imgs) + history
     history.append(
         {"role": "user", "content": [{"type": "text", "text": message}]}
     )
@@ -146,6 +166,11 @@ with gr.Blocks(title="Chat with a character via reference sheet!") as demo:
         gr.HTML(_("title"))
         img = gr.Image(type="filepath", value=default_img, label=_("upload"), render=False)
         description = gr.TextArea(value=_("default_description"), label=_("additional_description"), render=False)
+        more_imgs = gr.Files(
+            label=_("more_imgs"),
+            file_types=["image"],
+            render=False
+        )
         confirm_btn = gr.Button(_("confirm"), render=False)
         chatbot = gr.Chatbot(height=600, type='messages', label=_("chatbox"), render=False)
         engine = gr.Radio([(_('local'), 'local'), ('API', 'api')],
@@ -158,6 +183,7 @@ with gr.Blocks(title="Chat with a character via reference sheet!") as demo:
                 img.render()
                 with gr.Tab(_("description")):
                     description.render()
+                    more_imgs.render()
                 with gr.Tab(_("more_options")):
                     engine.render()
                     base_url.render()
@@ -169,9 +195,9 @@ with gr.Blocks(title="Chat with a character via reference sheet!") as demo:
                     response,
                     chatbot=chatbot,
                     type="messages",
-                    additional_inputs=[img, description, engine, base_url, api_model, api_key],
+                    additional_inputs=[img, description, more_imgs, engine, base_url, api_model, api_key],
                 )
-        confirm_btn.click(prefill_chatbot, [img, description, engine, base_url, api_model, api_key], chat.chatbot)\
+        confirm_btn.click(prefill_chatbot, [img, description, more_imgs, engine, base_url, api_model, api_key], chat.chatbot)\
             .then(lambda x: x, chat.chatbot, chat.chatbot_value)
 
 

readme.md

@@ -1,6 +1,6 @@
 ---
 title: Refsheet Chat
-emoji: 📉
+emoji: 💬
 colorFrom: gray
 colorTo: green
 sdk: gradio
@@ -11,7 +11,7 @@ license: mit
 short_description: Chat with a character via reference sheet!
 ---
 
-# Chat with Reference Sheet
+# Chat via Reference Sheet
 
 A demo of [Gemma 3](https://blog.google/technology/developers/gemma-3/), demonstrating its excellent vision and multilingual capability.