


import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class     __snake_case							(      _UpperCAmelCase  ,			unittest.TestCase):
					SCREAMING_SNAKE_CASE_		:					Dict 				= RealmTokenizer
					SCREAMING_SNAKE_CASE_		:					List[Any] 				= RealmTokenizerFast
					SCREAMING_SNAKE_CASE_		:					str 				= True
					SCREAMING_SNAKE_CASE_		:					str 				= True
					SCREAMING_SNAKE_CASE_		:					Any 				= filter_non_english
					def   a (	self					:	Tuple     )       ->				Tuple:



											'''simple docstring'''





											super().setUp()

											__magic_name__ 		=			[
											    '[UNK]',
											    '[CLS]',
											    '[SEP]',
											    '[PAD]',
											    '[MASK]',
											    'want',
											    '##want',
											    '##ed',
											    'wa',
											    'un',
											    'runn',
											    '##ing',
											    ',',
											    'low',
											    'lowest',
											]
											__magic_name__ 		=			os.path.join(self.tmpdirname  ,			VOCAB_FILES_NAMES['vocab_file']     )
											with open(self.vocab_file  ,			'w'  ,			encoding='utf-8'     ) as vocab_writer:
																	vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )
					def   a (	self					:	Tuple  ,			_lowerCAmelCase					:	Tuple     )       ->				Dict:



											'''simple docstring'''





											__magic_name__ 		=			'UNwant\u00E9d,running'
											__magic_name__ 		=			'unwanted, running'
											return input_text, output_text
					def   a (	self					:	str     )       ->				List[Any]:



											'''simple docstring'''





											__magic_name__ 		=			self.tokenizer_class(self.vocab_file     )

											__magic_name__ 		=			tokenizer.tokenize('UNwant\u00E9d,running'     )
											self.assertListEqual(_lowerCAmelCase  ,			['un', '##want', '##ed', ',', 'runn', '##ing']     )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase     )  ,			[9, 6, 7, 1_2, 1_0, 1_1]     )
					def   a (	self					:	Union[str, Any]     )       ->				List[Any]:



											'''simple docstring'''





											if not self.test_rust_tokenizer:
																	return

											__magic_name__ 		=			self.get_tokenizer()
											__magic_name__ 		=			self.get_rust_tokenizer()

											__magic_name__ 		=			'UNwant\u00E9d,running'

											__magic_name__ 		=			tokenizer.tokenize(_lowerCAmelCase     )
											__magic_name__ 		=			rust_tokenizer.tokenize(_lowerCAmelCase     )
											self.assertListEqual(_lowerCAmelCase  ,			_lowerCAmelCase     )

											__magic_name__ 		=			tokenizer.encode(_lowerCAmelCase  ,			add_special_tokens=_lowerCAmelCase     )
											__magic_name__ 		=			rust_tokenizer.encode(_lowerCAmelCase  ,			add_special_tokens=_lowerCAmelCase     )
											self.assertListEqual(_lowerCAmelCase  ,			_lowerCAmelCase     )

											__magic_name__ 		=			self.get_rust_tokenizer()
											__magic_name__ 		=			tokenizer.encode(_lowerCAmelCase     )
											__magic_name__ 		=			rust_tokenizer.encode(_lowerCAmelCase     )
											self.assertListEqual(_lowerCAmelCase  ,			_lowerCAmelCase     )

											# With lower casing
											__magic_name__ 		=			self.get_tokenizer(do_lower_case=_lowerCAmelCase     )
											__magic_name__ 		=			self.get_rust_tokenizer(do_lower_case=_lowerCAmelCase     )

											__magic_name__ 		=			'UNwant\u00E9d,running'

											__magic_name__ 		=			tokenizer.tokenize(_lowerCAmelCase     )
											__magic_name__ 		=			rust_tokenizer.tokenize(_lowerCAmelCase     )
											self.assertListEqual(_lowerCAmelCase  ,			_lowerCAmelCase     )

											__magic_name__ 		=			tokenizer.encode(_lowerCAmelCase  ,			add_special_tokens=_lowerCAmelCase     )
											__magic_name__ 		=			rust_tokenizer.encode(_lowerCAmelCase  ,			add_special_tokens=_lowerCAmelCase     )
											self.assertListEqual(_lowerCAmelCase  ,			_lowerCAmelCase     )

											__magic_name__ 		=			self.get_rust_tokenizer()
											__magic_name__ 		=			tokenizer.encode(_lowerCAmelCase     )
											__magic_name__ 		=			rust_tokenizer.encode(_lowerCAmelCase     )
											self.assertListEqual(_lowerCAmelCase  ,			_lowerCAmelCase     )
					def   a (	self					:	Union[str, Any]     )       ->				str:



											'''simple docstring'''





											__magic_name__ 		=			BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'     )  ,			['ah', '\u535A', '\u63A8', 'zz']     )
					def   a (	self					:	List[str]     )       ->				int:



											'''simple docstring'''





											__magic_name__ 		=			BasicTokenizer(do_lower_case=_lowerCAmelCase     )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '     )  ,			['hello', '!', 'how', 'are', 'you', '?']     )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )  ,			['hello']     )
					def   a (	self					:	Union[str, Any]     )       ->				Dict:



											'''simple docstring'''





											__magic_name__ 		=			BasicTokenizer(do_lower_case=_lowerCAmelCase  ,			strip_accents=_lowerCAmelCase     )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )  ,			['hällo', '!', 'how', 'are', 'you', '?']     )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )  ,			['h\u00E9llo']     )
					def   a (	self					:	List[str]     )       ->				int:



											'''simple docstring'''





											__magic_name__ 		=			BasicTokenizer(do_lower_case=_lowerCAmelCase  ,			strip_accents=_lowerCAmelCase     )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )  ,			['hallo', '!', 'how', 'are', 'you', '?']     )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )  ,			['hello']     )
					def   a (	self					:	List[str]     )       ->				Tuple:



											'''simple docstring'''





											__magic_name__ 		=			BasicTokenizer(do_lower_case=_lowerCAmelCase     )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )  ,			['hallo', '!', 'how', 'are', 'you', '?']     )
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'     )  ,			['hello']     )
					def   a (	self					:	Tuple     )       ->				Optional[int]:



											'''simple docstring'''





											__magic_name__ 		=			BasicTokenizer(do_lower_case=_lowerCAmelCase     )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '     )  ,			['HeLLo', '!', 'how', 'Are', 'yoU', '?']     )
					def   a (	self					:	int     )       ->				Union[str, Any]:



											'''simple docstring'''





											__magic_name__ 		=			BasicTokenizer(do_lower_case=_lowerCAmelCase  ,			strip_accents=_lowerCAmelCase     )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )  ,			['HäLLo', '!', 'how', 'Are', 'yoU', '?']     )
					def   a (	self					:	Optional[Any]     )       ->				Optional[int]:



											'''simple docstring'''





											__magic_name__ 		=			BasicTokenizer(do_lower_case=_lowerCAmelCase  ,			strip_accents=_lowerCAmelCase     )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '     )  ,			['HaLLo', '!', 'how', 'Are', 'yoU', '?']     )
					def   a (	self					:	int     )       ->				Dict:



											'''simple docstring'''





											__magic_name__ 		=			BasicTokenizer(do_lower_case=_lowerCAmelCase  ,			never_split=['[UNK]']     )

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'     )  ,			['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']     )
					def   a (	self					:	str     )       ->				int:



											'''simple docstring'''





											__magic_name__ 		=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

											__magic_name__ 		=			{}
											for i, token in enumerate(_lowerCAmelCase     ):
																	__magic_name__ 		=			i
											__magic_name__ 		=			WordpieceTokenizer(vocab=_lowerCAmelCase  ,			unk_token='[UNK]'     )

											self.assertListEqual(tokenizer.tokenize(''     )  ,			[]     )

											self.assertListEqual(tokenizer.tokenize('unwanted running'     )  ,			['un', '##want', '##ed', 'runn', '##ing']     )

											self.assertListEqual(tokenizer.tokenize('unwantedX running'     )  ,			['[UNK]', 'runn', '##ing']     )
					def   a (	self					:	Optional[Any]     )       ->				Any:



											'''simple docstring'''





											self.assertTrue(_is_whitespace(' '     )     )
											self.assertTrue(_is_whitespace('\t'     )     )
											self.assertTrue(_is_whitespace('\r'     )     )
											self.assertTrue(_is_whitespace('\n'     )     )
											self.assertTrue(_is_whitespace('\u00A0'     )     )

											self.assertFalse(_is_whitespace('A'     )     )
											self.assertFalse(_is_whitespace('-'     )     )
					def   a (	self					:	Optional[Any]     )       ->				Any:



											'''simple docstring'''





											self.assertTrue(_is_control('\u0005'     )     )

											self.assertFalse(_is_control('A'     )     )
											self.assertFalse(_is_control(' '     )     )
											self.assertFalse(_is_control('\t'     )     )
											self.assertFalse(_is_control('\r'     )     )
					def   a (	self					:	Union[str, Any]     )       ->				str:



											'''simple docstring'''





											self.assertTrue(_is_punctuation('-'     )     )
											self.assertTrue(_is_punctuation('$'     )     )
											self.assertTrue(_is_punctuation('`'     )     )
											self.assertTrue(_is_punctuation('.'     )     )

											self.assertFalse(_is_punctuation('A'     )     )
											self.assertFalse(_is_punctuation(' '     )     )
					def   a (	self					:	Union[str, Any]     )       ->				int:



											'''simple docstring'''





											__magic_name__ 		=			self.get_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(_lowerCAmelCase     ) for t in ['Test', '\xad', 'test']]  ,			[['[UNK]'], [], ['[UNK]']]     )

											if self.test_rust_tokenizer:
																	__magic_name__ 		=			self.get_rust_tokenizer()
																	self.assertListEqual(
																	    [rust_tokenizer.tokenize(_lowerCAmelCase     ) for t in ['Test', '\xad', 'test']]  ,			[['[UNK]'], [], ['[UNK]']]     )
					@slow
					def   a (	self					:	Any     )       ->				Union[str, Any]:



											'''simple docstring'''





											__magic_name__ 		=			self.tokenizer_class.from_pretrained('bert-base-uncased'     )

											__magic_name__ 		=			tokenizer.encode('sequence builders'  ,			add_special_tokens=_lowerCAmelCase     )
											__magic_name__ 		=			tokenizer.encode('multi-sequence build'  ,			add_special_tokens=_lowerCAmelCase     )

											__magic_name__ 		=			tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase     )
											__magic_name__ 		=			tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase  ,			_lowerCAmelCase     )

											assert encoded_sentence == [1_0_1] + text + [1_0_2]
											assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
					def   a (	self					:	Any     )       ->				Union[str, Any]:



											'''simple docstring'''





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
																							__magic_name__ 		=			self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,			**_lowerCAmelCase     )

																							__magic_name__ 		=			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																							__magic_name__ 		=			tokenizer_r.encode_plus(
																							    _lowerCAmelCase  ,			return_attention_mask=_lowerCAmelCase  ,			return_token_type_ids=_lowerCAmelCase  ,			return_offsets_mapping=_lowerCAmelCase  ,			add_special_tokens=_lowerCAmelCase  ,			)

																							__magic_name__ 		=			tokenizer_r.do_lower_case if hasattr(_lowerCAmelCase  ,			'do_lower_case'     ) else False
																							__magic_name__ 		=			(
																							    [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'A'),
																							        ((1, 2), ','),
																							        ((3, 5), 'na'),
																							        ((5, 6), '##ï'),
																							        ((6, 8), '##ve'),
																							        ((9, 1_5), tokenizer_r.mask_token),
																							        ((1_6, 2_1), 'Allen'),
																							        ((2_1, 2_3), '##NL'),
																							        ((2_3, 2_4), '##P'),
																							        ((2_5, 3_3), 'sentence'),
																							        ((3_3, 3_4), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							    if not do_lower_case
																							    else [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), 'a'),
																							        ((1, 2), ','),
																							        ((3, 8), 'naive'),
																							        ((9, 1_5), tokenizer_r.mask_token),
																							        ((1_6, 2_1), 'allen'),
																							        ((2_1, 2_3), '##nl'),
																							        ((2_3, 2_4), '##p'),
																							        ((2_5, 3_3), 'sentence'),
																							        ((3_3, 3_4), '.'),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							)

																							self.assertEqual(
																							    [e[1] for e in expected_results]  ,			tokenizer_r.convert_ids_to_tokens(tokens['input_ids']     )     )
																							self.assertEqual([e[0] for e in expected_results]  ,			tokens['offset_mapping']     )







					@slow
					def   a (	self					:	List[str]     )       ->				Union[str, Any]:



											'''simple docstring'''





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
																							__magic_name__ 		=			self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,			**_lowerCAmelCase     )
																							__magic_name__ 		=			self.tokenizer_class.from_pretrained(_lowerCAmelCase  ,			**_lowerCAmelCase     )
																							__magic_name__ 		=			[['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

																							__magic_name__ 		=			tokenizer_r.batch_encode_candidates(_lowerCAmelCase  ,			max_length=1_0  ,			return_tensors='np'     )
																							__magic_name__ 		=			tokenizer_p.batch_encode_candidates(_lowerCAmelCase  ,			max_length=1_0  ,			return_tensors='np'     )

																							__magic_name__ 		=			(2, 2, 1_0)

																							self.assertEqual(encoded_sentence_r['input_ids'].shape  ,			_lowerCAmelCase     )
																							self.assertEqual(encoded_sentence_r['attention_mask'].shape  ,			_lowerCAmelCase     )
																							self.assertEqual(encoded_sentence_r['token_type_ids'].shape  ,			_lowerCAmelCase     )

																							self.assertEqual(encoded_sentence_p['input_ids'].shape  ,			_lowerCAmelCase     )
																							self.assertEqual(encoded_sentence_p['attention_mask'].shape  ,			_lowerCAmelCase     )
																							self.assertEqual(encoded_sentence_p['token_type_ids'].shape  ,			_lowerCAmelCase     )





def 							UpperCamelCase_  (				__snake_case			:						int     )->      bool:




						'''simple docstring'''
						if not isinstance(__snake_case	,			__snake_case     ):
												__magic_name__ 		=			F'''Input value of [number={number}] must be an integer'''
												raise TypeError(__snake_case     )
						if number < 0 or number % 2 == 1:
												return False
						__magic_name__ 		=			int(number**0.5     )
						return number == number_sqrt * (number_sqrt + 1)


if __name__ == "__main__":
					import doctest

					doctest.testmod()

