oliverholworthy commited on
Commit
39aa015
·
unverified ·
1 Parent(s): 72dc0fb

docs: enable multimodal encoder compilation in vLLM examples

Browse files

Signed-off-by: Oliver Holworthy <oliverholworthy@users.noreply.huggingface.co>

Files changed (1) hide show
  1. README.md +2 -0
README.md CHANGED
@@ -374,6 +374,7 @@ def make_input(tokenizer, query, document):
374
  def main() -> None:
375
  model = LLM(
376
  model=MODEL_PATH,
 
377
  runner="pooling",
378
  dtype="bfloat16",
379
  limit_mm_per_prompt={"image": 1},
@@ -437,6 +438,7 @@ vllm serve "$MODEL_PATH" \
437
  --runner pooling \
438
  --dtype bfloat16 \
439
  --max-model-len 32768 \
 
440
  --max-num-seqs 4 \
441
  --limit-mm-per-prompt '{"image":1}' \
442
  --pooler-config '{"seq_pooling_type":"MEAN","use_activation":false}' \
 
374
  def main() -> None:
375
  model = LLM(
376
  model=MODEL_PATH,
377
+ compilation_config={"compile_mm_encoder": True},
378
  runner="pooling",
379
  dtype="bfloat16",
380
  limit_mm_per_prompt={"image": 1},
 
438
  --runner pooling \
439
  --dtype bfloat16 \
440
  --max-model-len 32768 \
441
+ --compilation-config '{"compile_mm_encoder":true}' \
442
  --max-num-seqs 4 \
443
  --limit-mm-per-prompt '{"image":1}' \
444
  --pooler-config '{"seq_pooling_type":"MEAN","use_activation":false}' \