Update README.md
Browse files
README.md
CHANGED
|
@@ -67,25 +67,26 @@ FlagEval (Libra)** is a comprehensive evaluation system and open platform for la
|
|
| 67 |
|
| 68 |
```bash
|
| 69 |
pip install modelscope
|
| 70 |
-
modelscope download --model Qwen/Qwen2-7B-Instruct --local_dir /
|
| 71 |
|
| 72 |
```
|
| 73 |
|
| 74 |
### Download FlagOS Image
|
| 75 |
|
| 76 |
```bash
|
| 77 |
-
docker pull harbor.baai.ac.cn/flagrelease-public/
|
| 78 |
```
|
| 79 |
|
| 80 |
### Start the inference service
|
| 81 |
|
| 82 |
```bash
|
| 83 |
#Container Startup
|
| 84 |
-
docker run -d -it --net=host --uts=host --ipc=host
|
| 85 |
--privileged=true --group-add video --shm-size 100gb --ulimit memlock=-1 \
|
| 86 |
--security-opt seccomp=unconfined --security-opt apparmor=unconfined --device=/dev/dri \
|
| 87 |
-
--device=/dev/mxcd -v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro -v /
|
| 88 |
-
--name
|
|
|
|
| 89 |
```
|
| 90 |
|
| 91 |
### Serve
|
|
@@ -103,7 +104,7 @@ flagscale serve qwen2
|
|
| 103 |
import openai
|
| 104 |
openai.api_key = "EMPTY"
|
| 105 |
openai.base_url = "http://<server_ip>:8000/v1/"
|
| 106 |
-
model = "/
|
| 107 |
messages = [
|
| 108 |
{"role": "system", "content": "You are a helpful assistant."},
|
| 109 |
{"role": "user", "content": "What's the weather like today?"}
|
|
|
|
| 67 |
|
| 68 |
```bash
|
| 69 |
pip install modelscope
|
| 70 |
+
modelscope download --model Qwen/Qwen2-7B-Instruct --local_dir /data/Qwen2-7B-Instruct
|
| 71 |
|
| 72 |
```
|
| 73 |
|
| 74 |
### Download FlagOS Image
|
| 75 |
|
| 76 |
```bash
|
| 77 |
+
docker pull harbor.baai.ac.cn/flagrelease-public/flagrelease-nvidia-release-model_qwen2-7b-instruct-tree_none-gems_3.0-scale_0.8.0-cx_none-python_3.12.3-torch_2.8.0-pcp_cuda12.9-gpu_nvidia004-arc_amd64-driver_535.183.06:2512151909
|
| 78 |
```
|
| 79 |
|
| 80 |
### Start the inference service
|
| 81 |
|
| 82 |
```bash
|
| 83 |
#Container Startup
|
| 84 |
+
docker run -d -it --net=host --uts=host --ipc=host -e USE_FLAGGEMS=1 --gpus all \
|
| 85 |
--privileged=true --group-add video --shm-size 100gb --ulimit memlock=-1 \
|
| 86 |
--security-opt seccomp=unconfined --security-opt apparmor=unconfined --device=/dev/dri \
|
| 87 |
+
--device=/dev/mxcd -v /usr/share/zoneinfo/Asia/Shanghai:/etc/localtime:ro -v /data:/data/\
|
| 88 |
+
--name flagos harbor.baai.ac.cn/flagrelease-public/flagrelease-nvidia-release-model_qwen2-7b-instruct-tree_none-gems_3.0-scale_0.8.0-cx_none-python_3.12.3-torch_2.8.0-pcp_cuda12.9-gpu_nvidia004-arc_amd64-driver_535.183.06:2512151909 bash
|
| 89 |
+
docker exec -it flagos /bin/bash
|
| 90 |
```
|
| 91 |
|
| 92 |
### Serve
|
|
|
|
| 104 |
import openai
|
| 105 |
openai.api_key = "EMPTY"
|
| 106 |
openai.base_url = "http://<server_ip>:8000/v1/"
|
| 107 |
+
model = "/data/Qwen2-7B-Instruct"
|
| 108 |
messages = [
|
| 109 |
{"role": "system", "content": "You are a helpful assistant."},
|
| 110 |
{"role": "user", "content": "What's the weather like today?"}
|