
'''simple docstring'''






import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase	=    get_tests_dir('''fixtures/spiece.model''')

@require_sentencepiece
@require_tokenizers
class 							_snake_case     (  __a						,    unittest.TestCase ):
	"""simple docstring"""





	_UpperCamelCase							:  str								=		ErnieMTokenizer
	_UpperCamelCase							:  str								=		False
	_UpperCamelCase							:  Dict								=		True
	_UpperCamelCase							:  Optional[Any]								=		False
	_UpperCamelCase							:  Tuple								=		False




	def 	__A      (						self				: str      ):
			super().setUp()

			# We have a SentencePiece fixture for testing
			lowerCAmelCase_   :       str    							=ErnieMTokenizer(SCREAMING_SNAKE_CASE__					,						unk_token='''<unk>'''					,						pad_token='''<pad>'''      )
			tokenizer.save_pretrained(self.tmpdirname      )




	def 	__A      (						self				: List[Any]					,						UpperCamelCase_				: Optional[Any]      ):
			lowerCAmelCase_   :       List[str]    							='''this is a test'''
			lowerCAmelCase_   :       Dict    							='''this is a test'''
			return input_text, output_text




	def 	__A      (						self				: Optional[Any]      ):
			lowerCAmelCase_   :       Optional[Any]    							='''<pad>'''
			lowerCAmelCase_   :       Tuple    							=0

			self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE__      )					,						SCREAMING_SNAKE_CASE__      )
			self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE__      )					,						SCREAMING_SNAKE_CASE__      )




	def 	__A      (						self				: List[Any]      ):
			lowerCAmelCase_   :       List[str]    							=list(self.get_tokenizer().get_vocab().keys()      )

			self.assertEqual(vocab_keys[0]					,						'''<pad>'''      )
			self.assertEqual(vocab_keys[1]					,						'''<unk>'''      )
			self.assertEqual(vocab_keys[-1]					,						'''▁eloquent'''      )
			self.assertEqual(len(SCREAMING_SNAKE_CASE__      )					,						30000      )




	def 	__A      (						self				: Optional[Any]      ):
			self.assertEqual(self.get_tokenizer().vocab_size					,						30000      )




	def 	__A      (						self				: Union[str, Any]      ):
			if not self.test_rust_tokenizer:
					return

			lowerCAmelCase_   :       List[Any]    							=self.get_tokenizer()
			lowerCAmelCase_   :       Dict    							=self.get_rust_tokenizer()

			lowerCAmelCase_   :       Optional[Any]    							='''I was born in 92000, and this is falsé.'''

			lowerCAmelCase_   :       List[str]    							=tokenizer.tokenize(SCREAMING_SNAKE_CASE__      )
			lowerCAmelCase_   :       List[Any]    							=rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE__      )
			self.assertListEqual(SCREAMING_SNAKE_CASE__					,						SCREAMING_SNAKE_CASE__      )

			lowerCAmelCase_   :       List[Any]    							=tokenizer.encode(SCREAMING_SNAKE_CASE__					,						add_special_tokens=SCREAMING_SNAKE_CASE__      )
			lowerCAmelCase_   :       Optional[Any]    							=rust_tokenizer.encode(SCREAMING_SNAKE_CASE__					,						add_special_tokens=SCREAMING_SNAKE_CASE__      )
			self.assertListEqual(SCREAMING_SNAKE_CASE__					,						SCREAMING_SNAKE_CASE__      )

			lowerCAmelCase_   :       Tuple    							=self.get_rust_tokenizer()
			lowerCAmelCase_   :       int    							=tokenizer.encode(SCREAMING_SNAKE_CASE__      )
			lowerCAmelCase_   :       Optional[Any]    							=rust_tokenizer.encode(SCREAMING_SNAKE_CASE__      )
			self.assertListEqual(SCREAMING_SNAKE_CASE__					,						SCREAMING_SNAKE_CASE__      )




	def 	__A      (						self				: Optional[Any]      ):
			lowerCAmelCase_   :       Tuple    							=ErnieMTokenizer(SCREAMING_SNAKE_CASE__					,						do_lower_case=SCREAMING_SNAKE_CASE__					,						unk_token='''<unk>'''					,						pad_token='''<pad>'''      )

			lowerCAmelCase_   :       int    							=tokenizer.tokenize('''This is a test'''      )
			self.assertListEqual(SCREAMING_SNAKE_CASE__					,						['''▁this''', '''▁is''', '''▁a''', '''▁test''']      )

			self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__      )					,						[48, 25, 21, 1289]      )

			lowerCAmelCase_   :       Dict    							=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
			# ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
			self.assertListEqual(
			    SCREAMING_SNAKE_CASE__					,						['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''']      )
			lowerCAmelCase_   :       str    							=tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__      )
			self.assertListEqual(SCREAMING_SNAKE_CASE__					,						[31, 23, 386, 19, 518, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]      )

			lowerCAmelCase_   :       Dict    							=tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__      )
			self.assertListEqual(
			    SCREAMING_SNAKE_CASE__					,						['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']					,						)




	def 	__A      (						self				: Optional[Any]      ):
			lowerCAmelCase_   :       str    							=ErnieMTokenizer(SCREAMING_SNAKE_CASE__					,						unk_token='''<unk>'''					,						pad_token='''<pad>'''      )

			lowerCAmelCase_   :       Any    							=tokenizer.encode('''sequence builders'''      )
			lowerCAmelCase_   :       Optional[Any]    							=tokenizer.encode('''multi-sequence build'''      )

			lowerCAmelCase_   :       List[str]    							=tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__      )
			lowerCAmelCase_   :       int    							=tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__					,						SCREAMING_SNAKE_CASE__      )

			assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
			assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
			    tokenizer.sep_token_id
			] + text_a + [tokenizer.sep_token_id]





	@slow
	def 	__A      (						self				: List[str]      ):
			# fmt: off
			lowerCAmelCase_   :       Tuple    							={'''input_ids''': [[0, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 9, 304, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 5, 5, 5, 16, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 6460, 1328, 4589, 42, 122009, 115774, 23, 3559, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [0, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
			# fmt: on

			self.tokenizer_integration_test_util(
			    expected_encoding=SCREAMING_SNAKE_CASE__					,						model_name='''susnato/ernie-m-base_pytorch'''					,						sequences=[
			        '''Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '''
			        '''general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural '''
			        '''Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained '''
			        '''models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.''',
			        '''BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '''
			        '''conditioning on both left and right context in all layers.''',
			        '''The quick brown fox jumps over the lazy dog.''',
			    ]					,						)









"""simple docstring"""




SCREAMING_SNAKE_CASE_  :      Optional[int]							=      {0: [1, 2], 1: [0, 3], 2: [0], 3: [1], 4: [5, 6], 5: [4, 6], 6: [4, 5]}

SCREAMING_SNAKE_CASE_  :      str							=      {0: [1, 2, 3], 1: [0, 3], 2: [0], 3: [0, 1], 4: [], 5: []}



def 						UpperCAmelCase__ (					A__						,     A__						,     A__ )   ->   list:




   """simple docstring"""
   lowerCamelCase__						       =      True
   lowerCamelCase__						       =      []

   for neighbour in graph[vert]:
      if not visited[neighbour]:
         connected_verts += dfs(A__						,     A__						,     A__ )

   return [vert, *connected_verts]



def 						UpperCAmelCase__ (					A__ )   ->   list:




   """simple docstring"""
   lowerCamelCase__						       =      len(A__ )
   lowerCamelCase__						       =      graph_size * [False]
   lowerCamelCase__						       =      []

   for i in range(A__ ):
      if not visited[i]:
         lowerCamelCase__						       =      dfs(A__						,     A__						,     A__ )
         components_list.append(A__ )

   return components_list


if __name__ == "__main__":
     import doctest

     doctest.testmod()




