aphrodite-engine/docker/docker-compose.yml

services:
  aphrodite-engine:
    build:
      context: .
    restart: on-failure:5
    environment:
      - NUM_GPUS=${NUM_GPUS}
      - MODEL_NAME=${MODEL_NAME}
      - REVISION=${REVISION}
      - HUGGING_FACE_HUB_TOKEN=${HUGGING_FACE_HUB_TOKEN}
      - QUANTIZATION=${QUANTIZATION}
      - KVCACHE=${KVCACHE}
      - API_KEY=${API_KEY}
      - CONTEXT_LENGTH=${CONTEXT_LENGTH}
      - GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION}
      - ENFORCE_EAGER=${ENFORCE_EAGER}
    volumes:
      - ${HOME}/.cache:/root/.cache/
    shm_size: 10g
    ports:
      - "7860:7860"
      - "2242:2242"
    deploy:
      resources:
        reservations:
          devices:
          - driver: nvidia
            count: all
            capabilities: [gpu]