.env 946 B

1234567891011121314151617181920212223
  1. #HUGGING_FACE_HUB_TOKEN=<secret>
  2. #HF_CACHE=~/.cache/huggingface
  3. #UID=1000
  4. #GID=0
  5. #TZ=UTC
  6. #PORT=5000
  7. #HOST=0.0.0.0
  8. #API_KEY=sk-example
  9. #SSL_KEYFILE=~/ssl/server.key
  10. #SSL_CERTFILE=~/ssl/server.crt
  11. #MODEL_NAME=mistralai/Mistral-7B-Instruct-v0.2
  12. #REVISION=main
  13. #DATATYPE=half # FP16. Recommended for quantization.
  14. #KVCACHE=fp8_e5m2 # It reduces the GPU memory footprint and boosts the performance. But it may cause slight accuracy drop.
  15. #CONTEXT_LENGTH=32768 # If unspecified, will be automatically derived from the model.
  16. #NUM_GPUS=1
  17. #GPU_MEMORY_UTILIZATION=0.8 # If you are running out of memory, consider decreasing 'gpu_memory_utilization' or enforcing eager mode.
  18. #QUANTIZATION=awq
  19. #ENFORCE_EAGER=true # If you are running out of memory, consider decreasing 'gpu_memory_utilization' or enforcing eager mode.
  20. #KOBOLD_API=true # use this to launch a kobold compatible server in addition to the OpenAI one
  21. #CMD_ADDITIONAL_ARGUMENTS="--seed 0"