mkd-hossain/keural-alpha-base-model

---

library_name: transformers model_name: keural-alpha

vllm run command

python -m vllm.entrypoints.openai.api_server
--model mkd-hossain/keural-alpha-chat-v0.1
--served-model-name keural-alpha-chat-v0.1
--dtype bfloat16
--tensor-parallel-size 2
--max-model-len 2048
--gpu-memory-utilization 0.9
--port 10008

respose command

HISTORY="System: You are Keural-Alpha, an AI assistant developed by MKD Co., Ltd." MODEL="keural-alpha-chat-v0.1"

while true; do read -p "User: " USER [[ "$USER" == "exit" ]] && break

PROMPT="$HISTORY\nUser: $USER\nAssistant:"

RESPONSE=$(curl -s http://localhost:10008/v1/completions
-H "Content-Type: application/json"
-d "{ "model": "$MODEL", "prompt": "$PROMPT", "max_tokens": 120, "temperature": 0.9, "top_p": 0.85, "repetition_penalty": 1.15, "stop": ["\nUser:"] }")

ASSISTANT=$(echo "$RESPONSE" | python -c "import sys,json; print(json.load(sys.stdin)['choices'][0]['text'].strip())")

echo "Assistant: $ASSISTANT"

HISTORY="$HISTORY\nUser: $USER\nAssistant: $ASSISTANT" done