mixtral-api-local-with-parallel

Paused

muryshev commited on May 15, 2024

Commit

e2aa54d

verified ·

1 Parent(s): 7ec6732

Update run.sh

Files changed (1) hide show

run.sh CHANGED Viewed

@@ -29,4 +29,4 @@ else
     echo "File already exists."
 fi
 cd /app
-./server -m "$MODEL_PATH" -c $CONTEXT --port $PORT --host 0.0.0.0 --n-gpu-layers $N_GPU_LAYERS --parallel $PARALLEL_NUMBER --cont-batching --path "/app/public"

     echo "File already exists."
 fi
 cd /app
+./server -m "$MODEL_PATH" -c $CONTEXT --port $PORT --host 0.0.0.0 --n-gpu-layers $N_GPU_LAYERS --parallel 4 --cont-batching --path "/app/public"