hysts/zephyr-7b

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 hysts
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

app.py

@@ -1,25 +1,127 @@
-from fastapi import FastAPI, Request
-from fastapi.responses import HTMLResponse
-from fastapi.staticfiles import StaticFiles
-from fastapi.templating import Jinja2Templates
+#!/usr/bin/env python
 
-app = FastAPI()
+import os
+from collections.abc import Iterator
+from threading import Thread
 
-app.mount("/static", StaticFiles(directory="static"), name="static")
+import gradio as gr
+import spaces
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 
-templates = Jinja2Templates(directory="templates")
+DESCRIPTION = "# Zephyr-7B beta"
 
-@app.get("/", response_class=HTMLResponse)
-async def greet_json(request: Request):
-    return templates.TemplateResponse(
-        request=request, name="index.html", context={}
-    )
+if not torch.cuda.is_available():
+    DESCRIPTION += "\n<p>Running on CPU 🥶 This demo does not work on CPU.</p>"
+
+MAX_MAX_NEW_TOKENS = 2048
+DEFAULT_MAX_NEW_TOKENS = 1024
+MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "4096"))
+
+if torch.cuda.is_available():
+    model_id = "HuggingFaceH4/zephyr-7b-beta"
+    model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map="auto")
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+
+@spaces.GPU
+def generate(
+    message: str,
+    chat_history: list[dict],
+    system_prompt: str = "",
+    max_new_tokens: int = 1024,
+    temperature: float = 0.7,
+    top_p: float = 0.95,
+    top_k: int = 50,
+    repetition_penalty: float = 1.0,
+) -> Iterator[str]:
+    conversation = [{"role": "system", "content": system_prompt}] if system_prompt else []
+    conversation += [*chat_history, {"role": "user", "content": message}]
 
-@app.get("/items/{id}", response_class=HTMLResponse)
-async def read_item(request: Request, id: str):
-    return templates.TemplateResponse(
-        request=request, name="item.html", context={"id": id}
+    input_ids = tokenizer.apply_chat_template(conversation, return_tensors="pt", add_generation_prompt=True)
+    if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH:
+        input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:]
+        gr.Warning(f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens.")
+    input_ids = input_ids.to(model.device)
+
+    streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
+    generate_kwargs = dict(
+        {"input_ids": input_ids},
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        do_sample=True,
+        top_p=top_p,
+        top_k=top_k,
+        temperature=temperature,
+        num_beams=1,
+        repetition_penalty=repetition_penalty,
     )
+    t = Thread(target=model.generate, kwargs=generate_kwargs)
+    t.start()
+
+    outputs = []
+    for text in streamer:
+        outputs.append(text)
+        yield "".join(outputs)
+
+
+demo = gr.ChatInterface(
+    fn=generate,
+    additional_inputs=[
+        gr.Textbox(
+            label="System prompt",
+            lines=6,
+            placeholder="You are a friendly chatbot who always responds in the style of a pirate.",
+        ),
+        gr.Slider(
+            label="Max new tokens",
+            minimum=1,
+            maximum=MAX_MAX_NEW_TOKENS,
+            step=1,
+            value=DEFAULT_MAX_NEW_TOKENS,
+        ),
+        gr.Slider(
+            label="Temperature",
+            minimum=0.1,
+            maximum=4.0,
+            step=0.1,
+            value=0.7,
+        ),
+        gr.Slider(
+            label="Top-p (nucleus sampling)",
+            minimum=0.05,
+            maximum=1.0,
+            step=0.05,
+            value=0.95,
+        ),
+        gr.Slider(
+            label="Top-k",
+            minimum=1,
+            maximum=1000,
+            step=1,
+            value=50,
+        ),
+        gr.Slider(
+            label="Repetition penalty",
+            minimum=1.0,
+            maximum=2.0,
+            step=0.05,
+            value=1.0,
+        ),
+    ],
+    stop_btn=None,
+    examples=[
+        ["Hello there! How are you doing?"],
+        ["Can you explain briefly to me what is the Python programming language?"],
+        ["Explain the plot of Cinderella in a sentence."],
+        ["How many hours does it take a man to eat a Helicopter?"],
+        ["Write a 100-word article on 'Benefits of Open-Source in AI research'"],
+    ],
+    type="messages",
+    description=DESCRIPTION,
+    css_paths="style.css",
+)
+
 
-# to run server
-# $ uvicorn app:app --host 0.0.0.0 --port 7860
+if __name__ == "__main__":
+    demo.queue(max_size=20).launch()

requirements.txt

@@ -1,4 +1,240 @@
-fastapi
-jinja2
-uvicorn[standard]
-
+# This file was autogenerated by uv via the following command:
+#    uv pip compile pyproject.toml -o requirements.txt
+accelerate==1.2.1
+    # via zephyr-7b (pyproject.toml)
+aiofiles==23.2.1
+    # via gradio
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.8.0
+    # via
+    #   gradio
+    #   httpx
+    #   starlette
+certifi==2024.12.14
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+charset-normalizer==3.4.1
+    # via requests
+click==8.1.8
+    # via
+    #   typer
+    #   uvicorn
+exceptiongroup==1.2.2
+    # via anyio
+fastapi==0.115.6
+    # via gradio
+ffmpy==0.5.0
+    # via gradio
+filelock==3.16.1
+    # via
+    #   huggingface-hub
+    #   torch
+    #   transformers
+    #   triton
+fsspec==2024.12.0
+    # via
+    #   gradio-client
+    #   huggingface-hub
+    #   torch
+gradio==5.12.0
+    # via
+    #   zephyr-7b (pyproject.toml)
+    #   spaces
+gradio-client==1.5.4
+    # via gradio
+h11==0.14.0
+    # via
+    #   httpcore
+    #   uvicorn
+hf-transfer==0.1.9
+    # via zephyr-7b (pyproject.toml)
+httpcore==1.0.7
+    # via httpx
+httpx==0.28.1
+    # via
+    #   gradio
+    #   gradio-client
+    #   safehttpx
+    #   spaces
+huggingface-hub==0.27.1
+    # via
+    #   accelerate
+    #   gradio
+    #   gradio-client
+    #   tokenizers
+    #   transformers
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+jinja2==3.1.5
+    # via
+    #   gradio
+    #   torch
+markdown-it-py==3.0.0
+    # via rich
+markupsafe==2.1.5
+    # via
+    #   gradio
+    #   jinja2
+mdurl==0.1.2
+    # via markdown-it-py
+mpmath==1.3.0
+    # via sympy
+networkx==3.4.2
+    # via torch
+numpy==2.2.1
+    # via
+    #   accelerate
+    #   gradio
+    #   pandas
+    #   transformers
+nvidia-cublas-cu12==12.1.3.1
+    # via
+    #   nvidia-cudnn-cu12
+    #   nvidia-cusolver-cu12
+    #   torch
+nvidia-cuda-cupti-cu12==12.1.105
+    # via torch
+nvidia-cuda-nvrtc-cu12==12.1.105
+    # via torch
+nvidia-cuda-runtime-cu12==12.1.105
+    # via torch
+nvidia-cudnn-cu12==9.1.0.70
+    # via torch
+nvidia-cufft-cu12==11.0.2.54
+    # via torch
+nvidia-curand-cu12==10.3.2.106
+    # via torch
+nvidia-cusolver-cu12==11.4.5.107
+    # via torch
+nvidia-cusparse-cu12==12.1.0.106
+    # via
+    #   nvidia-cusolver-cu12
+    #   torch
+nvidia-nccl-cu12==2.20.5
+    # via torch
+nvidia-nvjitlink-cu12==12.6.85
+    # via
+    #   nvidia-cusolver-cu12
+    #   nvidia-cusparse-cu12
+nvidia-nvtx-cu12==12.1.105
+    # via torch
+orjson==3.10.14
+    # via gradio
+packaging==24.2
+    # via
+    #   accelerate
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   spaces
+    #   transformers
+pandas==2.2.3
+    # via gradio
+pillow==11.1.0
+    # via gradio
+psutil==5.9.8
+    # via
+    #   accelerate
+    #   spaces
+pydantic==2.10.5
+    # via
+    #   fastapi
+    #   gradio
+    #   spaces
+pydantic-core==2.27.2
+    # via pydantic
+pydub==0.25.1
+    # via gradio
+pygments==2.19.1
+    # via rich
+python-dateutil==2.9.0.post0
+    # via pandas
+python-multipart==0.0.20
+    # via gradio
+pytz==2024.2
+    # via pandas
+pyyaml==6.0.2
+    # via
+    #   accelerate
+    #   gradio
+    #   huggingface-hub
+    #   transformers
+regex==2024.11.6
+    # via transformers
+requests==2.32.3
+    # via
+    #   huggingface-hub
+    #   spaces
+    #   transformers
+rich==13.9.4
+    # via typer
+ruff==0.9.1
+    # via gradio
+safehttpx==0.1.6
+    # via gradio
+safetensors==0.5.2
+    # via
+    #   accelerate
+    #   transformers
+semantic-version==2.10.0
+    # via gradio
+shellingham==1.5.4
+    # via typer
+six==1.17.0
+    # via python-dateutil
+sniffio==1.3.1
+    # via anyio
+spaces==0.32.0
+    # via zephyr-7b (pyproject.toml)
+starlette==0.41.3
+    # via
+    #   fastapi
+    #   gradio
+sympy==1.13.3
+    # via torch
+tokenizers==0.21.0
+    # via transformers
+tomlkit==0.13.2
+    # via gradio
+torch==2.4.0
+    # via
+    #   zephyr-7b (pyproject.toml)
+    #   accelerate
+tqdm==4.67.1
+    # via
+    #   huggingface-hub
+    #   transformers
+transformers==4.48.0
+    # via zephyr-7b (pyproject.toml)
+triton==3.0.0
+    # via torch
+typer==0.15.1
+    # via gradio
+typing-extensions==4.12.2
+    # via
+    #   anyio
+    #   fastapi
+    #   gradio
+    #   gradio-client
+    #   huggingface-hub
+    #   pydantic
+    #   pydantic-core
+    #   rich
+    #   spaces
+    #   torch
+    #   typer
+    #   uvicorn
+tzdata==2024.2
+    # via pandas
+urllib3==2.3.0
+    # via requests
+uvicorn==0.34.0
+    # via gradio
+websockets==14.1
+    # via gradio-client

static/styles.css

@@ -1,7 +0,0 @@
-h1 a {
-  color: red;
-}
-
-h2 {
-  color: red;
-}

style.css

@@ -0,0 +1,11 @@
+h1 {
+  text-align: center;
+  display: block;
+}
+
+#duplicate-button {
+  margin: auto;
+  color: white;
+  background: #1565c0;
+  border-radius: 100vh;
+}

templates/index.html

@@ -1,165 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Compliment Bot 💖</title>
-    <!-- <script src="https://cdn.jsdelivr.net/npm/@gradio/client@1.2.0/dist/index.min.js"></script> -->
-    <style>
-        body {
-            font-family: Arial, sans-serif;
-            display: flex;
-            justify-content: center;
-            align-items: center;
-            height: 100vh;
-            margin: 0;
-            background-color: #f0f0f0;
-        }
-        .container {
-            text-align: center;
-            background-color: white;
-            padding: 20px;
-            border-radius: 10px;
-            box-shadow: 0 0 10px rgba(0,0,0,0.1);
-            max-width: 400px;
-            width: 100%;
-        }
-        #headshot {
-            max-width: 300px;
-            max-height: 300px;
-            margin: 20px auto;
-            display: block;
-        }
-        #compliment {
-            font-size: 18px;
-            font-weight: bold;
-            color: #4a4a4a;
-            min-height: 50px;
-        }
-        .loader {
-            border: 5px solid #f3f3f3;
-            border-top: 5px solid #3498db;
-            border-radius: 50%;
-            width: 30px;
-            height: 30px;
-            animation: spin 1s linear infinite;
-            margin: 20px auto;
-            display: none;
-        }
-        @keyframes spin {
-            0% { transform: rotate(0deg); }
-            100% { transform: rotate(360deg); }
-        }
-        #uploadButton {
-            background-color: #4CAF50;
-            border: none;
-            color: white;
-            padding: 10px 20px;
-            text-align: center;
-            text-decoration: none;
-            display: inline-block;
-            font-size: 16px;
-            margin: 4px 2px;
-            cursor: pointer;
-            border-radius: 5px;
-        }
-    </style>
-</head>
-<body>
-    <div class="container">
-        <h1> Compliment Bot 💖</h1>
-        <input type="file" id="fileInput" accept="image/*" style="display: none;">
-        <button id="uploadButton">Upload New Headshot</button>
-        <br><br>
-        <img id="headshot" src="" alt="Your headshot" style="display:none;">
-        <div class="loader" id="loader"></div>
-        <p id="compliment"></p>
-    </div>
-
-    <script>
-        const fileInput = document.getElementById('fileInput');
-        const uploadButton = document.getElementById('uploadButton');
-        const headshot = document.getElementById('headshot');
-        const compliment = document.getElementById('compliment');
-        const loader = document.getElementById('loader');
-
-        const SYSTEM_PROMPT = `
-You are helpful assistant that gives the best compliments to people.
-You will be given a caption of someone's headshot.
-Based on that caption, provide a one sentence compliment to the person in the image.
-Make sure you compliment the person in the image and not any objects or scenery.
-Do NOT include any hashtags in your compliment or phrases like (emojis: dog, smiling face with heart-eyes, sun).
-
-Here are some examples of the desired behavior:
-
-Caption: a front view of a man who is smiling, there is a lighthouse in the background, there is a grassy area on the left that is green and curved. in the distance you can see the ocean and the shore. there is a grey and cloudy sky above the lighthouse and the trees.
-Compliment: Your smile is as bright as a lighthouse, lighting up the world around you. 🌟
-
-Caption: in a close-up, a blonde woman with short, wavy hair, is the focal point of the image. she's dressed in a dark brown turtleneck sweater, paired with a black hat and a black suit jacket. her lips are a vibrant red, and her eyes are a deep brown. in the background, a man with a black hat and a white shirt is visible.
-Compliment: You are the epitome of elegance and grace, with a style that is as timeless as your beauty. 💃🎩
-
-Conversation begins below:
-
-`
-
-        uploadButton.addEventListener('click', function() {
-            fileInput.click();
-        });
-
-        fileInput.addEventListener('change', function(e) {
-            const file = e.target.files[0];
-            if (file) {
-                const reader = new FileReader();
-                reader.onload = function(event) {
-                    headshot.src = event.target.result;
-                    headshot.style.display = 'block';
-                    generateCompliment(file);
-                }
-                reader.readAsDataURL(file);
-            }
-        });
-
-        async function generateCompliment(file) {
-            compliment.textContent = '';
-            loader.style.display = 'block';
-
-            try {
-                const client_lib = await import("https://cdn.jsdelivr.net/npm/@gradio/client@1.2.0/dist/index.min.js");
-                const Client = client_lib.Client;
-                const handle_file = client_lib.handle_file;
-                const captioning_space = await Client.connect("gokaygokay/SD3-Long-Captioner");
-                const llm_space = await Client.connect("hysts/zephyr-7b");
-
-                const caption = await captioning_space.predict("/create_captions_rich", { image: file });
-                
-                
-                console.info("Caption", caption.data);
-                
-                const submission = llm_space.submit("/chat", {
-                    system_prompt: SYSTEM_PROMPT,
-                    message: `Caption: ${caption.data}\nCompliment: `,
-                    max_new_tokens: 256,
-                    temperature: 0.7,
-                    top_p: 0.95,
-                    top_k: 50,
-                    repetition_penalty: 1,
-                }
-                )
-
-                for await (const msg of submission) {
-                    loader.style.display = 'none';
-                    if (msg.type === "data") {
-                        console.log("msg.data", msg.data);
-                        compliment.textContent = msg.data[0]
-                    }
-                }
-            } catch (error) {
-                console.error('Error:', error);
-                loader.style.display = 'none';
-                compliment.textContent = "Oops! We couldn't generate a compliment. You're still awesome though!"
-            }
-
-        }
-    </script>
-</body>
-</html>

templates/item.html

@@ -1,14 +0,0 @@
-<html>
-
-<head>
-  <title>Item Details</title>
-  <link href="{{ url_for('static', path='/styles.css') }}" rel="stylesheet">
-</head>
-
-<body>
-  <h1>Hello <a href="{{ url_for('read_item', id=id) }}">Item ID: {{ id }}</a></h1>
-
-  <h2>Hello...</h2>
-</body>
-
-</html>