|
@@ -238,6 +238,7 @@ class AphroditeEngine:
|
|
if not self.model_config.skip_tokenizer_init:
|
|
if not self.model_config.skip_tokenizer_init:
|
|
self.tokenizer = self._init_tokenizer()
|
|
self.tokenizer = self._init_tokenizer()
|
|
self.detokenizer = Detokenizer(self.tokenizer)
|
|
self.detokenizer = Detokenizer(self.tokenizer)
|
|
|
|
+ tokenizer_group = self.get_tokenizer_group()
|
|
else:
|
|
else:
|
|
self.tokenizer = None
|
|
self.tokenizer = None
|
|
self.detokenizer = None
|
|
self.detokenizer = None
|