File size: 1,074 Bytes
bcdf9fa |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
#!/bin/bash
#!/bin/bash
set -x
python ./_infer.py \
model.path=./checkpoints/model \
model.load_param=False \
data.path=./data/parquet/test.parquet \
data.output_path=./model_output/sft_pass@1.jsonl \
data.batch_size=32 data.n_samples=1 \
rollout.tensor_model_parallel_size=1 \
rollout.temperature=0.7 rollout.top_p=0.9 rollout.n=1 rollout.do_sample=True \
rollout.prompt_length=1200 rollout.response_length=512 \
rollout.enable_chunked_prefill=True \
+rollout.kv_cache_dtype=fp8_e5m2 \
rollout.max_model_len=1800 \
rollout.max_num_batched_tokens=1800 \
rollout.max_num_seqs=1 \
+model.trust_remote_code=True \
+rollout.kv_cache_block_size=16 \
+rollout.swap_space=16 \
rollout.gpu_memory_utilization=0.7
# python ./_infer.py \
# model.load_param=True \
# model.load_param_path="./checkpoints/merged_r1qwen14b/model.pt" \
# data.output_path="./model_output/sft_pass@1.jsonl" \
# data.n_samples=10\
# data.path="./data/parquet/test.parquet" \
# rollout.temperature=0.9\
# rollout.top_p=0.9 \
# rollout.n=1\ |