|
@@ -5,10 +5,15 @@ set -xe
|
|
|
cd /app/aphrodite-engine
|
|
|
source activate aphrodite-engine
|
|
|
echo 'Starting Aphrodite Engine API server...'
|
|
|
-python -u -m aphrodite.endpoints.api_server_kobold \
|
|
|
+CMD="python -u -m aphrodite.endpoints.kobold.api_server \
|
|
|
--host 0.0.0.0 \
|
|
|
- --port 2242 \
|
|
|
+ --port 7860 \
|
|
|
--model $MODEL_NAME \
|
|
|
--tensor-parallel-size $NUM_GPUS \
|
|
|
- --quantization $QUANTIZATION \
|
|
|
- --dtype $DATATYPE
|
|
|
+ --dtype $DATATYPE"
|
|
|
+
|
|
|
+if [ -n "$QUANTIZATION" ]; then
|
|
|
+ CMD="$CMD --quantization $QUANTIZATION"
|
|
|
+fi
|
|
|
+
|
|
|
+exec $CMD
|