Dockerfile

# vLLM with quantized Llama 7B model
# Use a specific version compatible with HF Spaces GPU drivers
FROM vllm/vllm-openai:v0.6.6.post1

# Set environment variables
ENV MODEL_NAME="TheBloke/Llama-2-7B-Chat-AWQ"
ENV HOST="0.0.0.0"
ENV PORT="7860"

# Expose the API port
EXPOSE 7860

# Run vLLM OpenAI-compatible server
ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
CMD ["--model", "TheBloke/Llama-2-7B-Chat-AWQ", \
     "--quantization", "awq", \
     "--host", "0.0.0.0", \
     "--port", "7860", \
     "--max-model-len", "4096"]