


"""simple docstring"""


from transformers import ProcessorMixin




class 	__UpperCamelCase    (    __UpperCAmelCase	):
					lowercase_						:							int			=     """AutoFeatureExtractor"""
					lowercase_						:							Tuple			=     """AutoTokenizer"""







"""simple docstring"""


import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir




@dataclass
class 	__UpperCamelCase    :
					lowercase_						:							str




class 	__UpperCamelCase    (    unittest.TestCase	):
					def 					UpperCAmelCase__   (       self       :  Union[str, Any]      )  ->					Optional[Any]:
											lowerCAmelCase      :List[str]				  = get_tests_dir('fixtures/test_sentencepiece.model'      )
											lowerCAmelCase      :Tuple				  = get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model'      )

											lowerCAmelCase      :List[str]				  = FakeOriginalTokenizer(vocab_file=UpperCAmelCase      )

											with warnings.catch_warnings(record=UpperCAmelCase      ) as w:
																	lowerCAmelCase      :Dict				  = SpmConverter(UpperCAmelCase      )
											self.assertEqual(len(UpperCAmelCase      )    ,   0      )

											lowerCAmelCase      :int				  = FakeOriginalTokenizer(vocab_file=UpperCAmelCase      )

											with warnings.catch_warnings(record=UpperCAmelCase      ) as w:
																	lowerCAmelCase      :List[str]				  = SpmConverter(UpperCAmelCase      )
											self.assertEqual(len(UpperCAmelCase      )    ,   1      )

											self.assertIn(
											    'The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'
											    ' which is not implemented in the fast tokenizers.'    ,   str(w[0].message      )    ,   )