aphrodite-engine/tests/tokenization/test_get_eos.py

from aphrodite.transformers_utils.config import try_get_generation_config
from aphrodite.transformers_utils.tokenizer import get_tokenizer


def test_get_llama3_eos_token():
    model_name = "meta-llama/Meta-Llama-3-8B-Instruct"

    tokenizer = get_tokenizer(model_name)
    assert tokenizer.eos_token_id == 128009

    generation_config = try_get_generation_config(model_name,
                                                  trust_remote_code=False)
    assert generation_config is not None
    assert generation_config.eos_token_id == [128001, 128009]


def test_get_blip2_eos_token():
    model_name = "Salesforce/blip2-opt-2.7b"

    tokenizer = get_tokenizer(model_name)
    assert tokenizer.eos_token_id == 2

    generation_config = try_get_generation_config(model_name,
                                                  trust_remote_code=False)
    assert generation_config is not None
    assert generation_config.eos_token_id == 50118