






"""simple docstring"""




from ..utils import DummyObject, requires_backends






class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[Any]    =      ['''tokenizers''']


 def __init__(  self	:		Any							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		Dict							)      -> Optional[int]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Dict    =      ['''tokenizers''']


 def __init__(  self	:		str							,		*UpperCAmelCase__	:		Optional[int]							,		**UpperCAmelCase__	:		List[str]							)      -> Dict:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Any    =      ['''tokenizers''']


 def __init__(  self	:		Any							,		*UpperCAmelCase__	:		Any							,		**UpperCAmelCase__	:		Union[str, Any]							)      -> int:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Optional[int]    =      ['''tokenizers''']


 def __init__(  self	:		Tuple							,		*UpperCAmelCase__	:		List[str]							,		**UpperCAmelCase__	:		Any							)      -> Optional[int]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[Any]    =      ['''tokenizers''']


 def __init__(  self	:		str							,		*UpperCAmelCase__	:		Optional[Any]							,		**UpperCAmelCase__	:		Optional[Any]							)      -> List[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[str]    =      ['''tokenizers''']


 def __init__(  self	:		List[Any]							,		*UpperCAmelCase__	:		Dict							,		**UpperCAmelCase__	:		List[str]							)      -> Tuple:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : int    =      ['''tokenizers''']


 def __init__(  self	:		int							,		*UpperCAmelCase__	:		int							,		**UpperCAmelCase__	:		Dict							)      -> List[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Dict    =      ['''tokenizers''']


 def __init__(  self	:		int							,		*UpperCAmelCase__	:		str							,		**UpperCAmelCase__	:		Optional[int]							)      -> Optional[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Optional[Any]    =      ['''tokenizers''']


 def __init__(  self	:		Tuple							,		*UpperCAmelCase__	:		int							,		**UpperCAmelCase__	:		Any							)      -> Union[str, Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : int    =      ['''tokenizers''']


 def __init__(  self	:		Union[str, Any]							,		*UpperCAmelCase__	:		Any							,		**UpperCAmelCase__	:		Tuple							)      -> Tuple:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[str]    =      ['''tokenizers''']


 def __init__(  self	:		Optional[Any]							,		*UpperCAmelCase__	:		int							,		**UpperCAmelCase__	:		Dict							)      -> List[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Union[str, Any]    =      ['''tokenizers''']


 def __init__(  self	:		List[Any]							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		Optional[Any]							)      -> Optional[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Tuple    =      ['''tokenizers''']


 def __init__(  self	:		List[str]							,		*UpperCAmelCase__	:		Union[str, Any]							,		**UpperCAmelCase__	:		List[str]							)      -> List[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Optional[int]    =      ['''tokenizers''']


 def __init__(  self	:		Dict							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		Dict							)      -> List[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Optional[Any]    =      ['''tokenizers''']


 def __init__(  self	:		Union[str, Any]							,		*UpperCAmelCase__	:		List[Any]							,		**UpperCAmelCase__	:		int							)      -> Optional[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Any    =      ['''tokenizers''']


 def __init__(  self	:		Dict							,		*UpperCAmelCase__	:		Optional[int]							,		**UpperCAmelCase__	:		Union[str, Any]							)      -> List[str]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : str    =      ['''tokenizers''']


 def __init__(  self	:		List[Any]							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		Optional[int]							)      -> List[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : str    =      ['''tokenizers''']


 def __init__(  self	:		int							,		*UpperCAmelCase__	:		int							,		**UpperCAmelCase__	:		Tuple							)      -> str:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Dict    =      ['''tokenizers''']


 def __init__(  self	:		Optional[Any]							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		Dict							)      -> str:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : int    =      ['''tokenizers''']


 def __init__(  self	:		Optional[int]							,		*UpperCAmelCase__	:		int							,		**UpperCAmelCase__	:		Union[str, Any]							)      -> Tuple:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Union[str, Any]    =      ['''tokenizers''']


 def __init__(  self	:		Dict							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		int							)      -> Optional[int]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Dict    =      ['''tokenizers''']


 def __init__(  self	:		Any							,		*UpperCAmelCase__	:		List[str]							,		**UpperCAmelCase__	:		Tuple							)      -> Optional[int]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[str]    =      ['''tokenizers''']


 def __init__(  self	:		Optional[Any]							,		*UpperCAmelCase__	:		Union[str, Any]							,		**UpperCAmelCase__	:		Any							)      -> Optional[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Dict    =      ['''tokenizers''']


 def __init__(  self	:		Tuple							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		Optional[Any]							)      -> int:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[Any]    =      ['''tokenizers''']


 def __init__(  self	:		List[Any]							,		*UpperCAmelCase__	:		Any							,		**UpperCAmelCase__	:		List[str]							)      -> int:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[Any]    =      ['''tokenizers''']


 def __init__(  self	:		Optional[int]							,		*UpperCAmelCase__	:		Optional[int]							,		**UpperCAmelCase__	:		Tuple							)      -> Any:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : str    =      ['''tokenizers''']


 def __init__(  self	:		Optional[int]							,		*UpperCAmelCase__	:		str							,		**UpperCAmelCase__	:		List[Any]							)      -> int:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Dict    =      ['''tokenizers''']


 def __init__(  self	:		Optional[Any]							,		*UpperCAmelCase__	:		List[Any]							,		**UpperCAmelCase__	:		int							)      -> Optional[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : str    =      ['''tokenizers''']


 def __init__(  self	:		Tuple							,		*UpperCAmelCase__	:		int							,		**UpperCAmelCase__	:		str							)      -> Tuple:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Tuple    =      ['''tokenizers''']


 def __init__(  self	:		Any							,		*UpperCAmelCase__	:		Optional[int]							,		**UpperCAmelCase__	:		List[str]							)      -> int:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Optional[int]    =      ['''tokenizers''']


 def __init__(  self	:		List[Any]							,		*UpperCAmelCase__	:		List[str]							,		**UpperCAmelCase__	:		str							)      -> Any:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[Any]    =      ['''tokenizers''']


 def __init__(  self	:		str							,		*UpperCAmelCase__	:		str							,		**UpperCAmelCase__	:		Optional[Any]							)      -> Optional[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : int    =      ['''tokenizers''']


 def __init__(  self	:		Tuple							,		*UpperCAmelCase__	:		Union[str, Any]							,		**UpperCAmelCase__	:		Optional[int]							)      -> Tuple:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[Any]    =      ['''tokenizers''']


 def __init__(  self	:		Union[str, Any]							,		*UpperCAmelCase__	:		Optional[Any]							,		**UpperCAmelCase__	:		List[str]							)      -> Tuple:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Optional[Any]    =      ['''tokenizers''']


 def __init__(  self	:		List[Any]							,		*UpperCAmelCase__	:		List[Any]							,		**UpperCAmelCase__	:		Any							)      -> Optional[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Any    =      ['''tokenizers''']


 def __init__(  self	:		Tuple							,		*UpperCAmelCase__	:		Any							,		**UpperCAmelCase__	:		Any							)      -> Optional[int]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Optional[Any]    =      ['''tokenizers''']


 def __init__(  self	:		Dict							,		*UpperCAmelCase__	:		List[str]							,		**UpperCAmelCase__	:		Tuple							)      -> List[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[Any]    =      ['''tokenizers''']


 def __init__(  self	:		Union[str, Any]							,		*UpperCAmelCase__	:		List[Any]							,		**UpperCAmelCase__	:		int							)      -> Optional[int]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[str]    =      ['''tokenizers''']


 def __init__(  self	:		Any							,		*UpperCAmelCase__	:		Optional[int]							,		**UpperCAmelCase__	:		List[Any]							)      -> Tuple:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[Any]    =      ['''tokenizers''']


 def __init__(  self	:		Tuple							,		*UpperCAmelCase__	:		Any							,		**UpperCAmelCase__	:		List[str]							)      -> List[Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Tuple    =      ['''tokenizers''']


 def __init__(  self	:		Tuple							,		*UpperCAmelCase__	:		Optional[int]							,		**UpperCAmelCase__	:		str							)      -> Union[str, Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : str    =      ['''tokenizers''']


 def __init__(  self	:		Optional[int]							,		*UpperCAmelCase__	:		Union[str, Any]							,		**UpperCAmelCase__	:		Optional[Any]							)      -> Any:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : int    =      ['''tokenizers''']


 def __init__(  self	:		List[str]							,		*UpperCAmelCase__	:		Optional[int]							,		**UpperCAmelCase__	:		str							)      -> Any:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Optional[int]    =      ['''tokenizers''']


 def __init__(  self	:		int							,		*UpperCAmelCase__	:		Union[str, Any]							,		**UpperCAmelCase__	:		Any							)      -> Any:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Union[str, Any]    =      ['''tokenizers''']


 def __init__(  self	:		Any							,		*UpperCAmelCase__	:		List[str]							,		**UpperCAmelCase__	:		Union[str, Any]							)      -> List[str]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Dict    =      ['''tokenizers''']


 def __init__(  self	:		Dict							,		*UpperCAmelCase__	:		int							,		**UpperCAmelCase__	:		int							)      -> str:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[str]    =      ['''tokenizers''']


 def __init__(  self	:		Union[str, Any]							,		*UpperCAmelCase__	:		List[Any]							,		**UpperCAmelCase__	:		Union[str, Any]							)      -> str:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[Any]    =      ['''tokenizers''']


 def __init__(  self	:		int							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		Optional[int]							)      -> int:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[str]    =      ['''tokenizers''']


 def __init__(  self	:		int							,		*UpperCAmelCase__	:		Optional[int]							,		**UpperCAmelCase__	:		Dict							)      -> Dict:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Optional[Any]    =      ['''tokenizers''']


 def __init__(  self	:		str							,		*UpperCAmelCase__	:		Optional[Any]							,		**UpperCAmelCase__	:		Union[str, Any]							)      -> Dict:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Tuple    =      ['''tokenizers''']


 def __init__(  self	:		List[str]							,		*UpperCAmelCase__	:		List[Any]							,		**UpperCAmelCase__	:		Dict							)      -> List[str]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Dict    =      ['''tokenizers''']


 def __init__(  self	:		Union[str, Any]							,		*UpperCAmelCase__	:		Dict							,		**UpperCAmelCase__	:		List[str]							)      -> List[str]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : int    =      ['''tokenizers''']


 def __init__(  self	:		Tuple							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		List[Any]							)      -> str:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[str]    =      ['''tokenizers''']


 def __init__(  self	:		Dict							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		List[Any]							)      -> Optional[int]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Tuple    =      ['''tokenizers''']


 def __init__(  self	:		Union[str, Any]							,		*UpperCAmelCase__	:		List[Any]							,		**UpperCAmelCase__	:		int							)      -> Any:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Union[str, Any]    =      ['''tokenizers''']


 def __init__(  self	:		str							,		*UpperCAmelCase__	:		Union[str, Any]							,		**UpperCAmelCase__	:		Optional[int]							)      -> Union[str, Any]:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : List[str]    =      ['''tokenizers''']


 def __init__(  self	:		str							,		*UpperCAmelCase__	:		Tuple							,		**UpperCAmelCase__	:		Tuple							)      -> int:
   requires_backends(self							,		["""tokenizers"""]							)







class  UpperCamelCase    (			metaclass=snake_case_   ):
 UpperCamelCase      : Dict    =      ['''tokenizers''']


 def __init__(  self	:		Union[str, Any]							,		*UpperCAmelCase__	:		List[str]							,		**UpperCAmelCase__	:		List[str]							)      -> List[Any]:
   requires_backends(self							,		["""tokenizers"""]							)









"""simple docstring"""




import os
import unittest

from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer

from ...test_tokenization_common import TokenizerTesterMixin






class  UpperCamelCase    (			snake_case_	,  unittest.TestCase   ):
 UpperCamelCase      : Union[str, Any]    =      BertweetTokenizer
 UpperCamelCase      : Optional[Any]    =      False


 def _lowercase			(  self	:		Dict							)      -> List[Any]:
   super().setUp()

   # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
   _a						: Tuple    =			["""I""", """m""", """V@@""", """R@@""", """r""", """e@@"""]
   _a						: str    =			dict(zip(UpperCAmelCase__							,		range(len(UpperCAmelCase__							)							)							)							)
   _a						: Dict    =			["""#version: 0.2""", """a m</w>"""]
   _a						: int    =			{"""unk_token""": """<unk>"""}

   _a						: Union[str, Any]    =			os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES["""vocab_file"""]							)
   _a						: List[Any]    =			os.path.join(self.tmpdirname							,		VOCAB_FILES_NAMES["""merges_file"""]							)
   with open(self.vocab_file							,		"""w"""							,		encoding="""utf-8"""							) as fp:
     for token in vocab_tokens:
       fp.write(f"""{token} {vocab_tokens[token]}\n"""							)
   with open(self.merges_file							,		"""w"""							,		encoding="""utf-8"""							) as fp:
     fp.write("""\n""".join(UpperCAmelCase__							)							)


 def _lowercase			(  self	:		Any							,		**UpperCAmelCase__	:		Optional[Any]							)      -> Dict:
   kwargs.update(self.special_tokens_map							)
   return BertweetTokenizer.from_pretrained(self.tmpdirname							,		**UpperCAmelCase__							)


 def _lowercase			(  self	:		Dict							,		UpperCAmelCase__	:		Optional[int]							)      -> Dict:
   _a						: List[str]    =			"""I am VinAI Research"""
   _a						: Optional[int]    =			"""I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"""
   return input_text, output_text







 def _lowercase			(  self	:		Tuple							)      -> Optional[int]:
   _a						: Optional[int]    =			BertweetTokenizer(self.vocab_file							,		self.merges_file							,		**self.special_tokens_map							)
   _a						: Tuple    =			"""I am VinAI Research"""
   _a						: Dict    =			"""I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h""".split()
   _a						: str    =			tokenizer.tokenize(UpperCAmelCase__							)
   self.assertListEqual(UpperCAmelCase__							,		UpperCAmelCase__							)

   _a						: Tuple    =			tokens + [tokenizer.unk_token]

   _a						: Tuple    =			[4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
   self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__							)							,		UpperCAmelCase__							)

