


'''simple docstring'''




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class _snake_case (			_SCREAMING_SNAKE_CASE     , unittest.TestCase ):
							"""simple docstring"""





							__a				       =  ElectraTokenizer
							__a				       =  ElectraTokenizerFast
							__a				       =  True
							__a				       =  True
							__a				       =  filter_non_english







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Optional[Any]:



											"""simple docstring"""







											super().setUp()

											__lowerCamelCase		:   Optional[int]           =       [
											    "[UNK]",
											    "[CLS]",
											    "[SEP]",
											    "[PAD]",
											    "[MASK]",
											    "want",
											    "##want",
											    "##ed",
											    "wa",
											    "un",
											    "runn",
											    "##ing",
											    ",",
											    "low",
											    "lowest",
											]
											__lowerCamelCase		:   Tuple           =       os.path.join(self.tmpdirname      ,					VOCAB_FILES_NAMES["vocab_file"]						)
											with open(self.vocab_file      ,					"w"      ,					encoding="utf-8"						) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)







							def 		__SCREAMING_SNAKE_CASE( self      ,					__A						)  ->Optional[Any]:



											"""simple docstring"""







											__lowerCamelCase		:   Optional[int]           =       "UNwant\u00E9d,running"
											__lowerCamelCase		:   Dict           =       "unwanted, running"
											return input_text, output_text







							def 		__SCREAMING_SNAKE_CASE( self						)  ->str:



											"""simple docstring"""







											__lowerCamelCase		:   Union[str, Any]           =       self.tokenizer_class(self.vocab_file						)

											__lowerCamelCase		:   Tuple           =       tokenizer.tokenize("UNwant\u00E9d,running"						)
											self.assertListEqual(A_      ,					["un", "##want", "##ed", ",", "runn", "##ing"]						)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(A_						)      ,					[9, 6, 7, 12, 10, 11]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Tuple:



											"""simple docstring"""







											if not self.test_rust_tokenizer:
															return

											__lowerCamelCase		:   str           =       self.get_tokenizer()
											__lowerCamelCase		:   Tuple           =       self.get_rust_tokenizer()

											__lowerCamelCase		:   Optional[int]           =       "UNwant\u00E9d,running"

											__lowerCamelCase		:   Optional[Any]           =       tokenizer.tokenize(A_						)
											__lowerCamelCase		:   Optional[int]           =       rust_tokenizer.tokenize(A_						)
											self.assertListEqual(A_      ,					A_						)

											__lowerCamelCase		:   Dict           =       tokenizer.encode(A_      ,					add_special_tokens=A_						)
											__lowerCamelCase		:   List[str]           =       rust_tokenizer.encode(A_      ,					add_special_tokens=A_						)
											self.assertListEqual(A_      ,					A_						)

											__lowerCamelCase		:   List[str]           =       self.get_rust_tokenizer()
											__lowerCamelCase		:   Optional[Any]           =       tokenizer.encode(A_						)
											__lowerCamelCase		:   Union[str, Any]           =       rust_tokenizer.encode(A_						)
											self.assertListEqual(A_      ,					A_						)

											# With lower casing
											__lowerCamelCase		:   List[str]           =       self.get_tokenizer(do_lower_case=A_						)
											__lowerCamelCase		:   Any           =       self.get_rust_tokenizer(do_lower_case=A_						)

											__lowerCamelCase		:   Optional[int]           =       "UNwant\u00E9d,running"

											__lowerCamelCase		:   Optional[Any]           =       tokenizer.tokenize(A_						)
											__lowerCamelCase		:   Tuple           =       rust_tokenizer.tokenize(A_						)
											self.assertListEqual(A_      ,					A_						)

											__lowerCamelCase		:   str           =       tokenizer.encode(A_      ,					add_special_tokens=A_						)
											__lowerCamelCase		:   Optional[int]           =       rust_tokenizer.encode(A_      ,					add_special_tokens=A_						)
											self.assertListEqual(A_      ,					A_						)

											__lowerCamelCase		:   Dict           =       self.get_rust_tokenizer()
											__lowerCamelCase		:   str           =       tokenizer.encode(A_						)
											__lowerCamelCase		:   Optional[int]           =       rust_tokenizer.encode(A_						)
											self.assertListEqual(A_      ,					A_						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Tuple:



											"""simple docstring"""







											__lowerCamelCase		:   Optional[Any]           =       BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"						)      ,					["ah", "\u535A", "\u63A8", "zz"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Optional[int]:



											"""simple docstring"""







											__lowerCamelCase		:   List[str]           =       BasicTokenizer(do_lower_case=A_						)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)      ,					["hello", "!", "how", "are", "you", "?"]						)
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)      ,					["hello"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->int:



											"""simple docstring"""







											__lowerCamelCase		:   Optional[int]           =       BasicTokenizer(do_lower_case=A_      ,					strip_accents=A_						)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)      ,					["hällo", "!", "how", "are", "you", "?"]						)
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)      ,					["h\u00E9llo"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->str:



											"""simple docstring"""







											__lowerCamelCase		:   Tuple           =       BasicTokenizer(do_lower_case=A_      ,					strip_accents=A_						)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)      ,					["hallo", "!", "how", "are", "you", "?"]						)
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)      ,					["hello"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->List[str]:



											"""simple docstring"""







											__lowerCamelCase		:   Any           =       BasicTokenizer(do_lower_case=A_						)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)      ,					["hallo", "!", "how", "are", "you", "?"]						)
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo"						)      ,					["hello"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Optional[Any]:



											"""simple docstring"""







											__lowerCamelCase		:   Any           =       BasicTokenizer(do_lower_case=A_						)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "						)      ,					["HeLLo", "!", "how", "Are", "yoU", "?"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->str:



											"""simple docstring"""







											__lowerCamelCase		:   Optional[int]           =       BasicTokenizer(do_lower_case=A_      ,					strip_accents=A_						)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)      ,					["HäLLo", "!", "how", "Are", "yoU", "?"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Union[str, Any]:



											"""simple docstring"""







											__lowerCamelCase		:   Dict           =       BasicTokenizer(do_lower_case=A_      ,					strip_accents=A_						)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "						)      ,					["HaLLo", "!", "how", "Are", "yoU", "?"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Tuple:



											"""simple docstring"""







											__lowerCamelCase		:   str           =       BasicTokenizer(do_lower_case=A_      ,					never_split=["[UNK]"]						)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"						)      ,					["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->List[Any]:



											"""simple docstring"""







											__lowerCamelCase		:   List[str]           =       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

											__lowerCamelCase		:   Union[str, Any]           =       {}
											for i, token in enumerate(A_						):
															__lowerCamelCase		:   str           =       i
											__lowerCamelCase		:   List[Any]           =       WordpieceTokenizer(vocab=A_      ,					unk_token="[UNK]"						)

											self.assertListEqual(tokenizer.tokenize(""						)      ,					[]						)

											self.assertListEqual(tokenizer.tokenize("unwanted running"						)      ,					["un", "##want", "##ed", "runn", "##ing"]						)

											self.assertListEqual(tokenizer.tokenize("unwantedX running"						)      ,					["[UNK]", "runn", "##ing"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Optional[int]:



											"""simple docstring"""







											self.assertTrue(_is_whitespace(" "						)						)
											self.assertTrue(_is_whitespace("\t"						)						)
											self.assertTrue(_is_whitespace("\r"						)						)
											self.assertTrue(_is_whitespace("\n"						)						)
											self.assertTrue(_is_whitespace("\u00A0"						)						)

											self.assertFalse(_is_whitespace("A"						)						)
											self.assertFalse(_is_whitespace("-"						)						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->int:



											"""simple docstring"""







											self.assertTrue(_is_control("\u0005"						)						)

											self.assertFalse(_is_control("A"						)						)
											self.assertFalse(_is_control(" "						)						)
											self.assertFalse(_is_control("\t"						)						)
											self.assertFalse(_is_control("\r"						)						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->str:



											"""simple docstring"""







											self.assertTrue(_is_punctuation("-"						)						)
											self.assertTrue(_is_punctuation("$"						)						)
											self.assertTrue(_is_punctuation("`"						)						)
											self.assertTrue(_is_punctuation("."						)						)

											self.assertFalse(_is_punctuation("A"						)						)
											self.assertFalse(_is_punctuation(" "						)						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Any:



											"""simple docstring"""







											__lowerCamelCase		:   Union[str, Any]           =       self.get_tokenizer()
											__lowerCamelCase		:   List[Any]           =       self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(A_						) for t in ["Test", "\xad", "test"]]      ,					[["[UNK]"], [], ["[UNK]"]]						)

											self.assertListEqual(
											    [rust_tokenizer.tokenize(A_						) for t in ["Test", "\xad", "test"]]      ,					[["[UNK]"], [], ["[UNK]"]]						)







							@slow
							def 		__SCREAMING_SNAKE_CASE( self						)  ->str:



											"""simple docstring"""







											__lowerCamelCase		:   Dict           =       self.tokenizer_class.from_pretrained("google/electra-base-discriminator"						)

											__lowerCamelCase		:   Optional[Any]           =       tokenizer.encode("sequence builders"      ,					add_special_tokens=A_						)
											__lowerCamelCase		:   List[Any]           =       tokenizer.encode("multi-sequence build"      ,					add_special_tokens=A_						)

											__lowerCamelCase		:   int           =       tokenizer.build_inputs_with_special_tokens(A_						)
											__lowerCamelCase		:   Optional[Any]           =       tokenizer.build_inputs_with_special_tokens(A_      ,					A_						)

											assert encoded_sentence == [101] + text + [102]
											assert encoded_pair == [101] + text + [102] + text_a + [102]







							def 		__SCREAMING_SNAKE_CASE( self						)  ->Optional[int]:



											"""simple docstring"""







											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'						):
																			__lowerCamelCase		:   Union[str, Any]           =       self.rust_tokenizer_class.from_pretrained(A_      ,					**A_						)

																			__lowerCamelCase		:   str           =       f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																			__lowerCamelCase		:   List[str]           =       tokenizer_r.encode_plus(
																			    A_      ,					return_attention_mask=A_      ,					return_token_type_ids=A_      ,					return_offsets_mapping=A_      ,					add_special_tokens=A_      ,					)

																			__lowerCamelCase		:   Any           =       tokenizer_r.do_lower_case if hasattr(A_      ,					"do_lower_case"						) else False
																			__lowerCamelCase		:   Dict           =       (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), "A"),
																			        ((1, 2), ","),
																			        ((3, 5), "na"),
																			        ((5, 6), "##ï"),
																			        ((6, 8), "##ve"),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), "Allen"),
																			        ((21, 23), "##NL"),
																			        ((23, 24), "##P"),
																			        ((25, 33), "sentence"),
																			        ((33, 34), "."),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), "a"),
																			        ((1, 2), ","),
																			        ((3, 8), "naive"),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), "allen"),
																			        ((21, 23), "##nl"),
																			        ((23, 24), "##p"),
																			        ((25, 33), "sentence"),
																			        ((33, 34), "."),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]      ,					tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]						)						)
																			self.assertEqual([e[0] for e in expected_results]      ,					tokens["offset_mapping"]						)







							def 		__SCREAMING_SNAKE_CASE( self						)  ->List[str]:



											"""simple docstring"""







											__lowerCamelCase		:   int           =       ["的", "人", "有"]
											__lowerCamelCase		:   Optional[int]           =       "".join(A_						)
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'						):
																			__lowerCamelCase		:   Any           =       True
																			__lowerCamelCase		:   List[Any]           =       self.tokenizer_class.from_pretrained(A_      ,					**A_						)
																			__lowerCamelCase		:   Dict           =       self.rust_tokenizer_class.from_pretrained(A_      ,					**A_						)

																			__lowerCamelCase		:   Optional[Any]           =       tokenizer_p.encode(A_      ,					add_special_tokens=A_						)
																			__lowerCamelCase		:   Optional[Any]           =       tokenizer_r.encode(A_      ,					add_special_tokens=A_						)

																			__lowerCamelCase		:   Any           =       tokenizer_r.convert_ids_to_tokens(A_						)
																			__lowerCamelCase		:   List[Any]           =       tokenizer_p.convert_ids_to_tokens(A_						)

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(A_      ,					A_						)
																			self.assertListEqual(A_      ,					A_						)

																			__lowerCamelCase		:   Union[str, Any]           =       False
																			__lowerCamelCase		:   Optional[int]           =       self.rust_tokenizer_class.from_pretrained(A_      ,					**A_						)
																			__lowerCamelCase		:   Tuple           =       self.tokenizer_class.from_pretrained(A_      ,					**A_						)

																			__lowerCamelCase		:   Optional[int]           =       tokenizer_r.encode(A_      ,					add_special_tokens=A_						)
																			__lowerCamelCase		:   Optional[int]           =       tokenizer_p.encode(A_      ,					add_special_tokens=A_						)

																			__lowerCamelCase		:   Optional[int]           =       tokenizer_r.convert_ids_to_tokens(A_						)
																			__lowerCamelCase		:   Optional[int]           =       tokenizer_p.convert_ids_to_tokens(A_						)

																			# it is expected that only the first Chinese character is not preceded by "##".
																			__lowerCamelCase		:   int           =       [
																			    f'##{token}' if idx != 0 else token for idx, token in enumerate(A_						)
																			]
																			self.assertListEqual(A_      ,					A_						)
																			self.assertListEqual(A_      ,					A_						)







def  A			(   lowercase					)  ->    int:
		'''simple docstring'''

		return int(lowercase					) if x - int(lowercase					) >= 0 else int(lowercase					) - 1


if __name__ == "__main__":
					import doctest

					doctest.testmod()
