Spaces:

codebyam
/

Llama-3.2-3B-Instruct-Q8_0-GGUF-DEMO

Sleeping

App Files Files Community

codebyam commited on Jul 9

Commit

26cf2e1

verified ·

1 Parent(s): 0e82c7c

Upload 3 files

Browse files

Files changed (3) hide show

Dockerfile +55 -0
app.py +398 -0
start.sh +14 -0

Dockerfile ADDED Viewed

	@@ -0,0 +1,55 @@

+FROM ubuntu:22.04
+# Install system dependencies
+RUN apt-get update && \
+    apt-get install -y \
+    build-essential \
+    libssl-dev \
+    zlib1g-dev \
+    libboost-math-dev \
+    libboost-python-dev \
+    libboost-timer-dev \
+    libboost-thread-dev \
+    libboost-system-dev \
+    libboost-filesystem-dev \
+    libopenblas-dev \
+    libomp-dev \
+    cmake \
+    pkg-config \
+    git \
+    python3-pip \
+    curl \
+    libcurl4-openssl-dev \
+    wget && \
+    rm -rf /var/lib/apt/lists/*
+# Build llama.cpp with OpenBLAS
+RUN git clone https://github.com/ggerganov/llama.cpp && \
+    cd llama.cpp && \
+    cmake -B build -S . \
+        -DLLAMA_BUILD_SERVER=ON \
+        -DLLAMA_BUILD_EXAMPLES=ON \
+        -DGGML_BLAS=ON \
+        -DGGML_BLAS_VENDOR=OpenBLAS \
+        -DCMAKE_BUILD_TYPE=Release && \
+    cmake --build build --config Release --target llama-server -j $(nproc)
+RUN cd /llama.cpp/build && ./bin/llama-server --list-devices
+# Download model
+RUN mkdir -p /models && \
+    wget -O /models/model.q8_0.gguf https://huggingface.co/hugging-quants/Llama-3.2-3B-Instruct-Q8_0-GGUF/resolve/main/llama-3.2-3b-instruct-q8_0.gguf
+RUN pip install fastapi uvicorn openai
+# Copy app and startup script
+COPY app.py /app.py
+COPY start.sh /start.sh
+RUN chmod +x /start.sh
+# Expose ports
+EXPOSE 7860 8080
+# Start services
+CMD ["/start.sh"]

app.py ADDED Viewed

	@@ -0,0 +1,398 @@

+from fastapi import FastAPI, Response, Cookie
+from fastapi.responses import HTMLResponse
+from pydantic import BaseModel, Field
+import time
+app = FastAPI()
+Tokens = []
+History = []
+@app.get("/", response_class=HTMLResponse)
+async def read_root(response: Response):
+    token = time.time()
+    Tokens.append(token)
+    response.set_cookie(key="token", value=token, httponly=True, samesite='strict')  # Set cookie
+    return '''<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Chatbot Assistant</title>
+    <script src="https://cdn.tailwindcss.com"></script>
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet">
+    <style>
+        body {
+            font-family: 'Inter', sans-serif;
+            background-color: #1a202c; /* Dark background for the body */
+            display: flex;
+            justify-content: center;
+            align-items: center;
+            min-height: 100vh;
+            margin: 0;
+            padding: 16px;
+            box-sizing: border-box;
+            color: #e2e8f0; /* Light text color for general body text */
+        }
+        .chat-container {
+            display: flex;
+            flex-direction: column;
+            width: 100%;
+            max-width: 800px;
+            height: 90vh;
+            background-color: #2d3748; /* Darker background for chat box */
+            border-radius: 1.5rem;
+            box-shadow: 0 20px 50px rgba(0, 0, 0, 0.3); /* More pronounced shadow for dark mode */
+            overflow: hidden;
+            border: 1px solid #4a5568; /* Subtle border for dark mode */
+        }
+        .chat-header {
+            background: linear-gradient(to right, #2c5282, #4c78a8); /* Darker blue gradient header */
+            color: #ffffff;
+            padding: 1.5rem;
+            text-align: center;
+            font-size: 1.75rem;
+            font-weight: 700;
+            border-top-left-radius: 1.5rem;
+            border-top-right-radius: 1.5rem;
+        }
+        .chat-messages {
+            flex-grow: 1;
+            padding: 1.5rem;
+            overflow-y: auto;
+            -webkit-overflow-scrolling: touch;
+            display: flex;
+            flex-direction: column;
+            gap: 1rem;
+        }
+        .message {
+            max-width: 75%;
+            padding: 0.85rem 1.25rem;
+            border-radius: 1.25rem;
+            word-wrap: break-word;
+            box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1); /* Subtle shadow for messages */
+            animation: fadeIn 0.3s ease-out;
+        }
+        .user-message {
+            background-color: #4299e1; /* Blue for user messages in dark mode */
+            align-self: flex-end;
+            color: #ffffff; /* White text for user messages */
+            border-bottom-right-radius: 0.5rem;
+        }
+        .assistant-message {
+            background-color: #4a5568; /* Darker gray for assistant messages */
+            align-self: flex-start;
+            color: #e2e8f0; /* Light text for assistant messages */
+            border-bottom-left-radius: 0.5rem;
+        }
+        .input-area {
+            display: flex;
+            align-items: center;
+            padding: 1.5rem;
+            border-top: 1px solid #4a5568; /* Darker border at the top */
+            background-color: #2d3748; /* Match chat box background */
+            gap: 1rem;
+        }
+        .input-area textarea {
+            flex-grow: 1;
+            padding: 0.85rem 1.25rem;
+            border: 2px solid #718096; /* Lighter border for contrast */
+            border-radius: 1.25rem;
+            resize: none;
+            outline: none;
+            font-size: 1rem;
+            line-height: 1.5;
+            min-height: 48px;
+            max-height: 150px;
+            overflow-y: auto;
+            transition: border-color 0.2s ease-in-out, box-shadow 0.2s ease-in-out;
+            background-color: #2d3748; /* Dark background for textarea */
+            color: #e2e8f0; /* Light text color for textarea */
+        }
+        .input-area textarea:focus {
+            border-color: #63b3ed; /* Lighter blue border on focus */
+            box-shadow: 0 0 0 3px rgba(99, 179, 237, 0.2); /* Lighter blue shadow on focus */
+        }
+        .input-area button {
+            background: linear-gradient(to right, #4299e1, #3182ce); /* Blue gradient button */
+            color: #ffffff;
+            padding: 0.85rem 1.75rem;
+            border-radius: 1.25rem;
+            font-weight: 600;
+            cursor: pointer;
+            transition: all 0.2s ease-in-out;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            gap: 0.5rem;
+            box-shadow: 0 4px 10px rgba(66, 153, 225, 0.3);
+            transform: translateY(0);
+        }
+        .input-area button:hover {
+            background: linear-gradient(to right, #3182ce, #2b6cb0);
+            box-shadow: 0 6px 15px rgba(66, 153, 225, 0.4);
+            transform: translateY(-2px);
+        }
+        .input-area button:disabled {
+            background: #4a5568; /* Dark gray when disabled */
+            cursor: not-allowed;
+            box-shadow: none;
+            transform: translateY(0);
+        }
+        /* Summarize button specific styles for dark mode */
+        #summarize-button {
+            background: linear-gradient(to right, #805ad5, #6b46c1); /* Darker purple gradient */
+            box-shadow: 0 4px 10px rgba(128, 90, 213, 0.3);
+        }
+        #summarize-button:hover {
+            background: linear-gradient(to right, #6b46c1, #553c9a);
+            box-shadow: 0 6px 15px rgba(128, 90, 213, 0.4);
+        }
+        .loading-indicator {
+            display: flex;
+            align-items: center;
+            gap: 0.5rem;
+            font-style: italic;
+            color: #a0aec0; /* Lighter gray for loading text */
+            align-self: flex-start;
+            animation: fadeIn 0.3s ease-out;
+        }
+        .loading-dot {
+            width: 8px;
+            height: 8px;
+            background-color: #a0aec0; /* Lighter gray for dots */
+            border-radius: 50%;
+            animation: bounce 1.4s infinite ease-in-out both;
+        }
+        .loading-dot:nth-child(1) { animation-delay: -0.32s; }
+        .loading-dot:nth-child(2) { animation-delay: -0.16s; }
+        .loading-dot:nth-child(3) { animation-delay: 0s; }
+        @keyframes bounce {
+            0%, 80%, 100% { transform: scale(0); }
+            40% { transform: scale(1); }
+        }
+        @keyframes fadeIn {
+            from { opacity: 0; transform: translateY(10px); }
+            to { opacity: 1; transform: translateY(0); }
+        }
+        /* Responsive adjustments (same as before, unaffected by dark mode colors) */
+        @media (max-width: 768px) {
+            body {
+                padding: 0;
+            }
+            .chat-container {
+                height: 100vh;
+                border-radius: 0;
+                box-shadow: none;
+                border: none;
+            }
+            .chat-header {
+                border-radius: 0;
+                font-size: 1.5rem;
+                padding: 1rem;
+            }
+            .chat-messages {
+                padding: 1rem;
+                gap: 0.75rem;
+            }
+            .message {
+                max-width: 85%;
+                padding: 0.75rem 1rem;
+                border-radius: 1rem;
+            }
+            .input-area {
+                flex-direction: column;
+                padding: 1rem;
+                gap: 0.75rem;
+            }
+            .input-area textarea {
+                width: 100%;
+                min-height: 40px;
+                padding: 0.75rem 1rem;
+                border-radius: 1rem;
+            }
+            .input-area button {
+                width: 100%;
+                padding: 0.75rem 1rem;
+                border-radius: 1rem;
+            }
+        }
+    </style>
+</head>
+<body class="antialiased">
+    <div class="chat-container">
+        <div class="chat-header">
+            Chat Assistant
+        </div>
+        <div id="chat-messages" class="chat-messages">
+        </div>
+        <div class="input-area">
+            <textarea
+                id="user-input"
+                placeholder="Type your message..."
+                rows="1"
+                class="shadow-sm block w-full"
+            ></textarea>
+            <button id="send-button">
+                <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
+                    <path d="M10.894 2.553a1 1 0 00-1.788 0l-7 14a1 1 0 001.169 1.409l5-1.429A1 1 0 009 15.571V11a1 1 0 112 0v4.571a1 1 0 00.553.894l5 1.429a1 1 0 001.169-1.409l-7-14z" />
+                </svg>
+                Send
+            </button>
+            <button id="summarize-button" style="display:none;">
+                <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
+                  <path fill-rule="evenodd" d="M4 4a2 2 0 012-2h4.586A2 2 0 0113 3.414L16.586 7A2 2 0 0117 8.414V16a2 2 0 01-2 2H6a2 2 0 01-2-2V4zm2 6a1 1 0 011-1h6a1 1 0 110 2H7a1 1 0 01-1-1zm1-3a1 1 0 000 2h.01a1 1 0 000-2H7zm0 6a1 1 0 000 2h.01a1 1 0 000-2H7zm3-6a1 1 0 000 2h3a1 1 0 100-2h-3zm0 6a1 1 0 000 2h3a1 1 0 100-2h-3z" clip-rule="evenodd" />
+                </svg>
+                Summarize Chat
+            </button>
+        </div>
+    </div>
+    <script>
+        const DOM = {
+            chatMessages: document.getElementById('chat-messages'),
+            userInput: document.getElementById('user-input'),
+            sendButton: document.getElementById('send-button'),
+            summarizeButton: document.getElementById('summarize-button'),
+        };
+        const chatHistory = [];
+        /**
+         * Adds a message to the chat display and updates chat history.
+         * @param {string} text - The message content.
+         * @param {'user'|'assistant'|'initial-assistant'} sender - The sender of the message.
+         */
+        function addMessage(text, sender) {
+            const messageDiv = document.createElement('div');
+            messageDiv.classList.add('message', sender === 'user' ? 'user-message' : 'assistant-message');
+            messageDiv.textContent = text;
+            DOM.chatMessages.appendChild(messageDiv);
+            DOM.chatMessages.scrollTop = DOM.chatMessages.scrollHeight;
+            if (sender !== 'initial-assistant') {
+                chatHistory.push({ role: sender, parts: [{ text: text }] });
+            }
+        }
+        /** Shows a loading indicator in the chat. */
+        function showLoadingIndicator() {
+            const loadingDiv = document.createElement('div');
+            loadingDiv.id = 'loading-indicator';
+            loadingDiv.classList.add('loading-indicator');
+            loadingDiv.innerHTML = `
+                <span>Assistant is typing</span>
+                <div class="loading-dot"></div>
+                <div class="loading-dot"></div>
+                <div class="loading-dot"></div>
+            `;
+            DOM.chatMessages.appendChild(loadingDiv);
+            DOM.chatMessages.scrollTop = DOM.chatMessages.scrollHeight;
+        }
+        /** Removes the loading indicator from the chat. */
+        function removeLoadingIndicator() {
+            document.getElementById('loading-indicator')?.remove();
+        }
+        /** Sets the disabled state of input and buttons. */
+        function setControlsDisabled(disabled) {
+            DOM.sendButton.disabled = disabled;
+            DOM.summarizeButton.disabled = disabled;
+            DOM.userInput.disabled = disabled;
+        }
+        /** Sends a user message to the backend API. */
+        async function sendMessage() {
+            const message = DOM.userInput.value.trim();
+            if (!message) return;
+            addMessage(message, 'user');
+            DOM.userInput.value = '';
+            setControlsDisabled(true);
+            showLoadingIndicator();
+            try {
+                const response = await fetch(`/response`, {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ prompt: message }),
+                });
+                if (!response.ok) {
+                    throw new Error(`HTTP error! status: ${response.status}`);
+                }
+                const data = await response.json();
+                addMessage(data.text || "Sorry, I couldn't get a response.", 'assistant');
+            } catch (error) {
+                console.error('Error sending message:', error);
+                addMessage('Error: Could not connect to the assistant. Please try again.', 'assistant');
+            } finally {
+                removeLoadingIndicator();
+                setControlsDisabled(false);
+                DOM.userInput.focus();
+                adjustTextareaHeight();
+            }
+        }
+        /** Dynamically adjusts the textarea height. */
+        function adjustTextareaHeight() {
+            DOM.userInput.style.height = 'auto';
+            DOM.userInput.style.height = (DOM.userInput.scrollHeight) + 'px';
+        }
+        // Event Listeners
+        DOM.sendButton.addEventListener('click', sendMessage);
+        DOM.summarizeButton.addEventListener('click', summarizeChat);
+        DOM.userInput.addEventListener('keypress', (event) => {
+            if (event.key === 'Enter' && !event.shiftKey) {
+                event.preventDefault();
+                sendMessage();
+            }
+        });
+        DOM.userInput.addEventListener('input', adjustTextareaHeight);
+        // Initial setup
+        document.addEventListener('DOMContentLoaded', () => {
+            addMessage('Hello! How can I assist you today?', 'initial-assistant');
+            DOM.userInput.focus();
+            adjustTextareaHeight(); // Set initial height for empty textarea
+        });
+    </script>
+</body>
+</html>'''
+from openai import OpenAI
+client = OpenAI(base_url="http://localhost:8080/v1", api_key="no-key-required")
+class ChatRequest(BaseModel):
+    """Request model for the chat endpoint."""
+    prompt: str
+@app.post("/response")
+async def handle_chat(chat_request: ChatRequest, token: str = Cookie(None)):
+    if token in Tokens:
+        i = Tokens.index(token)
+        History[i].append({"role": "user", "content": chat_request.prompt})
+        stream = client.chat.completions.create(
+                model="",
+                messages=History[i],
+            )
+        History[i].append({"role": "assistant", "content": chat_request.prompt})
+        return stream.choices[0].message.content
+    else: return 'Please stop. Just refresh the page.'
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=7860)

start.sh ADDED Viewed

	@@ -0,0 +1,14 @@

+#!/bin/bash
+# Start llama-server in background
+cd /llama.cpp/build
+./bin/llama-server --host 0.0.0.0 --port 8080 --model /models/model.q8_0.gguf --ctx-size 8192 --threads 2 &
+# Wait for server to initialize
+echo "Waiting for server to start..."
+until curl -s "http://localhost:8080/v1/models" >/dev/null; do
+    sleep 1
+done
+cd /
+python3 app.py