docker-compose-batch.yml

services:
  masp:
    image: masp_inf2_b16
    deploy:
    cpuset: "0-31"
    container_name: masp
    ports:
      - 8000:8000
    devices:
      - /dev/neuron0
      - /dev/neuron1
    volumes:
      - /root/llava_mistral_0531/inf2_dir_0531/inf2_weights:/root/inf2_dir/inf2_weights
      - /root/llava_mistral_0531/inf2_dir_0531/llava-mistral_videollava_092:/root/inf2_dir/llava-mistral_videollava_092
      - /root/llava_mistral_0531/app/llava:/app/llava
      - /root/llava_mistral_0531/inf2_dir_0531/mistral-compiled-bf16-b16:/root/inf2_dir/mistral-compiled-bf16-b16

  masp2:
    image: masp_inf2_b16
    deploy:
    cpuset: "32-63"
    container_name: masp2
    ports:
      - 8001:8000
    devices:
      - /dev/neuron2
      - /dev/neuron3
    volumes:
      - /root/llava_mistral_0531/inf2_dir_0531/inf2_weights:/root/inf2_dir/inf2_weights
      - /root/llava_mistral_0531/inf2_dir_0531/llava-mistral_videollava_092:/root/inf2_dir/llava-mistral_videollava_092
      - /root/llava_mistral_0531/app/llava:/app/llava
      - /root/llava_mistral_0531/inf2_dir_0531/mistral-compiled-bf16-b16:/root/inf2_dir/mistral-compiled-bf16-b16

  masp3:
    image: masp_inf2_b16
    deploy:
    cpuset: "64-95"
    container_name: masp3
    ports:
      - 8002:8000
    devices:
      - /dev/neuron4
      - /dev/neuron5
    volumes:
      - /root/llava_mistral_0531/inf2_dir_0531/inf2_weights:/root/inf2_dir/inf2_weights
      - /root/llava_mistral_0531/inf2_dir_0531/llava-mistral_videollava_092:/root/inf2_dir/llava-mistral_videollava_092
      - /root/llava_mistral_0531/app/llava:/app/llava
      - /root/llava_mistral_0531/inf2_dir_0531/mistral-compiled-bf16-b16:/root/inf2_dir/mistral-compiled-bf16-b16

  masp4:
    image: masp_inf2_b16
    deploy:
    cpuset: "96-127"
    container_name: masp4
    ports:
      - 8003:8000
    devices:
      - /dev/neuron6
      - /dev/neuron7
    volumes:
      - /root/llava_mistral_0531/inf2_dir_0531/inf2_weights:/root/inf2_dir/inf2_weights
      - /root/llava_mistral_0531/inf2_dir_0531/llava-mistral_videollava_092:/root/inf2_dir/llava-mistral_videollava_092
      - /root/llava_mistral_0531/app/llava:/app/llava
      - /root/llava_mistral_0531/inf2_dir_0531/mistral-compiled-bf16-b16:/root/inf2_dir/mistral-compiled-bf16-b16

  masp5:
    image: masp_inf2_b16
    deploy:
    cpuset: "128-159"
    container_name: masp5
    ports:
      - 8004:8000
    devices:
      - /dev/neuron8
      - /dev/neuron9
    volumes:
      - /root/llava_mistral_0531/inf2_dir_0531/inf2_weights:/root/inf2_dir/inf2_weights
      - /root/llava_mistral_0531/inf2_dir_0531/llava-mistral_videollava_092:/root/inf2_dir/llava-mistral_videollava_092
      - /root/llava_mistral_0531/app/llava:/app/llava
      - /root/llava_mistral_0531/inf2_dir_0531/mistral-compiled-bf16-b16:/root/inf2_dir/mistral-compiled-bf16-b16

  masp6:
    image: masp_inf2_b16
    deploy:
    cpuset: "160-191"
    container_name: masp6
    ports:
      - 8005:8000
    devices:
      - /dev/neuron10
      - /dev/neuron11
    volumes:
      - /root/llava_mistral_0531/inf2_dir_0531/inf2_weights:/root/inf2_dir/inf2_weights
      - /root/llava_mistral_0531/inf2_dir_0531/llava-mistral_videollava_092:/root/inf2_dir/llava-mistral_videollava_092
      - /root/llava_mistral_0531/app/llava:/app/llava
      - /root/llava_mistral_0531/inf2_dir_0531/mistral-compiled-bf16-b16:/root/inf2_dir/mistral-compiled-bf16-b16