




'''simple docstring'''





def 					UpperCamelCase						(	snake_case ):




   '''simple docstring'''

   if len(snake_case ) == 1:
      return a[0], a[0]
   else:
      # split A into half.
      SCREAMING_SNAKE_CASE__      :					List[str]							    =			a[: len(snake_case ) // 2]
      SCREAMING_SNAKE_CASE__      :					str							    =			a[len(snake_case ) // 2 :]

      # 2 sub problems, 1/2 of original size.
      SCREAMING_SNAKE_CASE__    ,					SCREAMING_SNAKE_CASE__      :					Tuple							    =			max_difference(snake_case )
      SCREAMING_SNAKE_CASE__    ,					SCREAMING_SNAKE_CASE__      :					List[Any]							    =			max_difference(snake_case )

      # get min of first and max of second
      # linear time
      SCREAMING_SNAKE_CASE__      :					str							    =			min(snake_case )
      SCREAMING_SNAKE_CASE__      :					Any							    =			max(snake_case )

      # 3 cases, either (small1, big1),
      # (min_first, max_second), (small2, big2)
      # constant comparisons
      if biga - smalla > max_second - min_first and biga - smalla > biga - smalla:
         return smalla, biga
      elif biga - smalla > max_second - min_first:
         return smalla, biga
      else:
         return min_first, max_second


if __name__ == "__main__":
       import doctest

       doctest.testmod()







import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class        snake_case__				(_UpperCamelCase       ,			unittest.TestCase  ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_			:				List[Any]									=   ElectraTokenizer
				SCREAMING_SNAKE_CASE_			:				int									=   ElectraTokenizerFast
				SCREAMING_SNAKE_CASE_			:				Union[str, Any]									=   True
				SCREAMING_SNAKE_CASE_			:				Tuple									=   True
				SCREAMING_SNAKE_CASE_			:				Optional[Any]									=   filter_non_english





				def 				__UpperCAmelCase				( self							:	Tuple				)   ->	List[Any]:
							super().setUp()

							a       =     [
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "[PAD]",
							    "[MASK]",
							    "want",
							    "##want",
							    "##ed",
							    "wa",
							    "un",
							    "runn",
							    "##ing",
							    ",",
							    "low",
							    "lowest",
							]
							a       =     os.path.join(self.tmpdirname   , VOCAB_FILES_NAMES["vocab_file"]				)
							with open(self.vocab_file   , "w"   , encoding="utf-8"				) as vocab_writer:
										vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)





				def 				__UpperCAmelCase				( self							:	List[Any]   , __lowerCamelCase							:	List[str]				)   ->	Union[str, Any]:
							a       =     "UNwant\u00E9d,running"
							a       =     "unwanted, running"
							return input_text, output_text





				def 				__UpperCAmelCase				( self							:	int				)   ->	Dict:
							a       =     self.tokenizer_class(self.vocab_file				)

							a       =     tokenizer.tokenize("UNwant\u00E9d,running"				)
							self.assertListEqual(__lowerCamelCase   , ["un", "##want", "##ed", ",", "runn", "##ing"]				)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase				)   , [9, 6, 7, 12, 10, 11]				)





				def 				__UpperCAmelCase				( self							:	List[Any]				)   ->	Optional[int]:
							if not self.test_rust_tokenizer:
										return

							a       =     self.get_tokenizer()
							a       =     self.get_rust_tokenizer()

							a       =     "UNwant\u00E9d,running"

							a       =     tokenizer.tokenize(__lowerCamelCase				)
							a       =     rust_tokenizer.tokenize(__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase   , __lowerCamelCase				)

							a       =     tokenizer.encode(__lowerCamelCase   , add_special_tokens=__lowerCamelCase				)
							a       =     rust_tokenizer.encode(__lowerCamelCase   , add_special_tokens=__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase   , __lowerCamelCase				)

							a       =     self.get_rust_tokenizer()
							a       =     tokenizer.encode(__lowerCamelCase				)
							a       =     rust_tokenizer.encode(__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase   , __lowerCamelCase				)

							# With lower casing
							a       =     self.get_tokenizer(do_lower_case=__lowerCamelCase				)
							a       =     self.get_rust_tokenizer(do_lower_case=__lowerCamelCase				)

							a       =     "UNwant\u00E9d,running"

							a       =     tokenizer.tokenize(__lowerCamelCase				)
							a       =     rust_tokenizer.tokenize(__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase   , __lowerCamelCase				)

							a       =     tokenizer.encode(__lowerCamelCase   , add_special_tokens=__lowerCamelCase				)
							a       =     rust_tokenizer.encode(__lowerCamelCase   , add_special_tokens=__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase   , __lowerCamelCase				)

							a       =     self.get_rust_tokenizer()
							a       =     tokenizer.encode(__lowerCamelCase				)
							a       =     rust_tokenizer.encode(__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase   , __lowerCamelCase				)





				def 				__UpperCAmelCase				( self							:	Union[str, Any]				)   ->	List[str]:
							a       =     BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)   , ["ah", "\u535A", "\u63A8", "zz"]				)





				def 				__UpperCAmelCase				( self							:	str				)   ->	Union[str, Any]:
							a       =     BasicTokenizer(do_lower_case=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)   , ["hello", "!", "how", "are", "you", "?"]				)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)   , ["hello"]				)





				def 				__UpperCAmelCase				( self							:	Dict				)   ->	str:
							a       =     BasicTokenizer(do_lower_case=__lowerCamelCase   , strip_accents=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)   , ["hällo", "!", "how", "are", "you", "?"]				)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)   , ["h\u00E9llo"]				)





				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	int:
							a       =     BasicTokenizer(do_lower_case=__lowerCamelCase   , strip_accents=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)   , ["hallo", "!", "how", "are", "you", "?"]				)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)   , ["hello"]				)





				def 				__UpperCAmelCase				( self							:	Tuple				)   ->	int:
							a       =     BasicTokenizer(do_lower_case=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)   , ["hallo", "!", "how", "are", "you", "?"]				)
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)   , ["hello"]				)





				def 				__UpperCAmelCase				( self							:	List[str]				)   ->	str:
							a       =     BasicTokenizer(do_lower_case=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)   , ["HeLLo", "!", "how", "Are", "yoU", "?"]				)





				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	Optional[int]:
							a       =     BasicTokenizer(do_lower_case=__lowerCamelCase   , strip_accents=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)   , ["HäLLo", "!", "how", "Are", "yoU", "?"]				)





				def 				__UpperCAmelCase				( self							:	int				)   ->	Optional[int]:
							a       =     BasicTokenizer(do_lower_case=__lowerCamelCase   , strip_accents=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)   , ["HaLLo", "!", "how", "Are", "yoU", "?"]				)





				def 				__UpperCAmelCase				( self							:	List[Any]				)   ->	str:
							a       =     BasicTokenizer(do_lower_case=__lowerCamelCase   , never_split=["[UNK]"]				)

							self.assertListEqual(
							    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)   , ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)





				def 				__UpperCAmelCase				( self							:	List[str]				)   ->	Optional[Any]:
							a       =     ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

							a       =     {}
							for i, token in enumerate(__lowerCamelCase				):
										a       =     i
							a       =     WordpieceTokenizer(vocab=__lowerCamelCase   , unk_token="[UNK]"				)

							self.assertListEqual(tokenizer.tokenize(""				)   , []				)

							self.assertListEqual(tokenizer.tokenize("unwanted running"				)   , ["un", "##want", "##ed", "runn", "##ing"]				)

							self.assertListEqual(tokenizer.tokenize("unwantedX running"				)   , ["[UNK]", "runn", "##ing"]				)





				def 				__UpperCAmelCase				( self							:	Optional[int]				)   ->	int:
							self.assertTrue(_is_whitespace(" "				)				)
							self.assertTrue(_is_whitespace("\t"				)				)
							self.assertTrue(_is_whitespace("\r"				)				)
							self.assertTrue(_is_whitespace("\n"				)				)
							self.assertTrue(_is_whitespace("\u00A0"				)				)

							self.assertFalse(_is_whitespace("A"				)				)
							self.assertFalse(_is_whitespace("-"				)				)





				def 				__UpperCAmelCase				( self							:	Union[str, Any]				)   ->	Optional[Any]:
							self.assertTrue(_is_control("\u0005"				)				)

							self.assertFalse(_is_control("A"				)				)
							self.assertFalse(_is_control(" "				)				)
							self.assertFalse(_is_control("\t"				)				)
							self.assertFalse(_is_control("\r"				)				)





				def 				__UpperCAmelCase				( self							:	int				)   ->	Any:
							self.assertTrue(_is_punctuation("-"				)				)
							self.assertTrue(_is_punctuation("$"				)				)
							self.assertTrue(_is_punctuation("`"				)				)
							self.assertTrue(_is_punctuation("."				)				)

							self.assertFalse(_is_punctuation("A"				)				)
							self.assertFalse(_is_punctuation(" "				)				)





				def 				__UpperCAmelCase				( self							:	List[Any]				)   ->	Optional[Any]:
							a       =     self.get_tokenizer()
							a       =     self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__lowerCamelCase				) for t in ["Test", "\xad", "test"]]   , [["[UNK]"], [], ["[UNK]"]]				)

							self.assertListEqual(
							    [rust_tokenizer.tokenize(__lowerCamelCase				) for t in ["Test", "\xad", "test"]]   , [["[UNK]"], [], ["[UNK]"]]				)





				@slow
				def 				__UpperCAmelCase				( self							:	Union[str, Any]				)   ->	Any:
							a       =     self.tokenizer_class.from_pretrained("google/electra-base-discriminator"				)

							a       =     tokenizer.encode("sequence builders"   , add_special_tokens=__lowerCamelCase				)
							a       =     tokenizer.encode("multi-sequence build"   , add_special_tokens=__lowerCamelCase				)

							a       =     tokenizer.build_inputs_with_special_tokens(__lowerCamelCase				)
							a       =     tokenizer.build_inputs_with_special_tokens(__lowerCamelCase   , __lowerCamelCase				)

							assert encoded_sentence == [1_01] + text + [1_02]
							assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]





				def 				__UpperCAmelCase				( self							:	str				)   ->	Union[str, Any]:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
													a       =     self.rust_tokenizer_class.from_pretrained(__lowerCamelCase   , **__lowerCamelCase				)

													a       =     f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
													a       =     tokenizer_r.encode_plus(
													    __lowerCamelCase   , return_attention_mask=__lowerCamelCase   , return_token_type_ids=__lowerCamelCase   , return_offsets_mapping=__lowerCamelCase   , add_special_tokens=__lowerCamelCase   , )

													a       =     tokenizer_r.do_lower_case if hasattr(__lowerCamelCase   , "do_lower_case"				) else False
													a       =     (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "A"),
													        ((1, 2), ","),
													        ((3, 5), "na"),
													        ((5, 6), "##ï"),
													        ((6, 8), "##ve"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "Allen"),
													        ((21, 23), "##NL"),
													        ((23, 24), "##P"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), "a"),
													        ((1, 2), ","),
													        ((3, 8), "naive"),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), "allen"),
													        ((21, 23), "##nl"),
													        ((23, 24), "##p"),
													        ((25, 33), "sentence"),
													        ((33, 34), "."),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]   , tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
													self.assertEqual([e[0] for e in expected_results]   , tokens["offset_mapping"]				)





				def 				__UpperCAmelCase				( self							:	Optional[Any]				)   ->	Union[str, Any]:
							a       =     ["的", "人", "有"]
							a       =     "".join(__lowerCamelCase				)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
													a       =     True
													a       =     self.tokenizer_class.from_pretrained(__lowerCamelCase   , **__lowerCamelCase				)
													a       =     self.rust_tokenizer_class.from_pretrained(__lowerCamelCase   , **__lowerCamelCase				)

													a       =     tokenizer_p.encode(__lowerCamelCase   , add_special_tokens=__lowerCamelCase				)
													a       =     tokenizer_r.encode(__lowerCamelCase   , add_special_tokens=__lowerCamelCase				)

													a       =     tokenizer_r.convert_ids_to_tokens(__lowerCamelCase				)
													a       =     tokenizer_p.convert_ids_to_tokens(__lowerCamelCase				)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(__lowerCamelCase   , __lowerCamelCase				)
													self.assertListEqual(__lowerCamelCase   , __lowerCamelCase				)

													a       =     False
													a       =     self.rust_tokenizer_class.from_pretrained(__lowerCamelCase   , **__lowerCamelCase				)
													a       =     self.tokenizer_class.from_pretrained(__lowerCamelCase   , **__lowerCamelCase				)

													a       =     tokenizer_r.encode(__lowerCamelCase   , add_special_tokens=__lowerCamelCase				)
													a       =     tokenizer_p.encode(__lowerCamelCase   , add_special_tokens=__lowerCamelCase				)

													a       =     tokenizer_r.convert_ids_to_tokens(__lowerCamelCase				)
													a       =     tokenizer_p.convert_ids_to_tokens(__lowerCamelCase				)

													# it is expected that only the first Chinese character is not preceded by "##".
													a       =     [
													    f"""##{token}""" if idx != 0 else token for idx, token in enumerate(__lowerCamelCase				)
													]
													self.assertListEqual(__lowerCamelCase   , __lowerCamelCase				)
													self.assertListEqual(__lowerCamelCase   , __lowerCamelCase				)






