#HUGGING_FACE_HUB_TOKEN=<secret>

#HF_CACHE=~/.cache/huggingface
#UID=1000
#GID=0
#TZ=UTC

#ENDPOINT=openai
#PORT=5000
#API_KEY=sk-example # ENDPOINT != kobold
#SSL_KEYFILE=~/ssl/server.key # ENDPOINT != kobold
#SSL_CERTFILE=~/ssl/server.crt # ENDPOINT != kobold
#MODEL_NAME=mistralai/Mistral-7B-Instruct-v0.2
#REVISION=main
#DATATYPE=half # FP16. Recommended for AWQ quantization.
#KVCACHE=fp8_e5m2 # It reduces the GPU memory footprint and boosts the performance. But it may cause slight accuracy drop.
#CONTEXT_LENGTH=32768 # If unspecified, will be automatically derived from the model.
#NUM_GPUS=1
#GPU_MEMORY_UTILIZATION=0.8 # If you are running out of memory, consider decreasing 'gpu_memory_utilization' or enforcing eager mode.
#QUANTIZATION=awq
#ENFORCE_EAGER=true # If you are running out of memory, consider decreasing 'gpu_memory_utilization' or enforcing eager mode.
#CMD_ADDITIONAL_ARGUMENTS="--seed 0"