Create run.sh

run.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+N_GPU_LAYERS="${N_GPU_LAYERS:-10}"
+MODEL_PATH="${MODEL_PATH:-/data/models/mixtral-8x7b-instruct-v0.1.Q2_K.gguf}"
+MODEL_URL="${MODEL_URL:-https://huggingface.co/TheBloke/Mixtral-8x7B-Instruct-v0.1-GGUF/resolve/main/mixtral-8x7b-instruct-v0.1.Q2_K.gguf}"
+PORT="${PORT:-7860}"
+CONTEXT="${CONTEXT:-32768}"
+
+# Function to print download progress
+print_progress() {
+    local percentage=$1
+    echo "Download progress: $percentage%"
+}
+
+calculate_progress() {
+    local current=$1
+    local total=$2
+    echo "scale=2; $current / $total * 100" | bc
+}
+
+# Check if the file exists
+if [ ! -e "$MODEL_PATH" ]; then
+    echo "Downloading model."
+    wget -q -O "$MODEL_PATH" "$MODEL_URL"
+    echo "Model downloaded."
+else
+    echo "File already exists."
+fi
+cd /app
+./server -m "$MODEL_PATH" -c $CONTEXT --port $PORT --host 0.0.0.0 --n-gpu-layers $N_GPU_LAYERS --path "/app/public"