embedding_inference.py 497 B

1234567891011121314151617
  1. from aphrodite import LLM
  2. # Sample prompts.
  3. prompts = [
  4. "Once upon a time,",
  5. "In a galaxy far, far away,",
  6. "The quick brown fox jumps over the lazy dog.",
  7. "The meaning of life is",
  8. ]
  9. # Create an LLM.
  10. model = LLM(model="intfloat/e5-mistral-7b-instruct", enforce_eager=True)
  11. # Generate embedding. The output is a list of EmbeddingRequestOutputs.
  12. outputs = model.encode(prompts)
  13. # Print the outputs.
  14. for output in outputs:
  15. print(output.outputs.embedding) # list of 4096 floats