




'''simple docstring'''





import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class     UpperCamelCase					(  a_			,		unittest.TestCase					):
		a    :Dict      					=				ElectraTokenizer
		a    :Optional[int]      					=				ElectraTokenizerFast
		a    :int      					=				True
		a    :Any      					=				True
		a    :Tuple      					=				filter_non_english



		def 						a_       ( self   ):







						'''simple docstring'''


						super().setUp()

						_a							:       Optional[int]       					=			[
						    '''[UNK]''',
						    '''[CLS]''',
						    '''[SEP]''',
						    '''[PAD]''',
						    '''[MASK]''',
						    '''want''',
						    '''##want''',
						    '''##ed''',
						    '''wa''',
						    '''un''',
						    '''runn''',
						    '''##ing''',
						    ''',''',
						    '''low''',
						    '''lowest''',
						]
						_a							:       Optional[Any]       					=			os.path.join(self.tmpdirname		,    VOCAB_FILES_NAMES["""vocab_file"""]   )
						with open(self.vocab_file		,    """w"""		,    encoding="""utf-8"""   ) as vocab_writer:
										vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]   )   )



		def 						a_       ( self		,    __lowerCAmelCase   ):







						'''simple docstring'''


						_a							:       int       					=			'''UNwant\u00E9d,running'''
						_a							:       Optional[Any]       					=			'''unwanted, running'''
						return input_text, output_text



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       List[str]       					=			self.tokenizer_class(self.vocab_file   )

						_a							:       Optional[int]       					=			tokenizer.tokenize("""UNwant\u00E9d,running"""   )
						self.assertListEqual(_snake_case		,    ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]   )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case   )		,    [9, 6, 7, 12, 10, 11]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						if not self.test_rust_tokenizer:
										return

						_a							:       Dict       					=			self.get_tokenizer()
						_a							:       int       					=			self.get_rust_tokenizer()

						_a							:       Union[str, Any]       					=			'''UNwant\u00E9d,running'''

						_a							:       List[str]       					=			tokenizer.tokenize(_snake_case   )
						_a							:       Optional[Any]       					=			rust_tokenizer.tokenize(_snake_case   )
						self.assertListEqual(_snake_case		,    _snake_case   )

						_a							:       Union[str, Any]       					=			tokenizer.encode(_snake_case		,    add_special_tokens=_snake_case   )
						_a							:       Optional[Any]       					=			rust_tokenizer.encode(_snake_case		,    add_special_tokens=_snake_case   )
						self.assertListEqual(_snake_case		,    _snake_case   )

						_a							:       Any       					=			self.get_rust_tokenizer()
						_a							:       List[Any]       					=			tokenizer.encode(_snake_case   )
						_a							:       List[str]       					=			rust_tokenizer.encode(_snake_case   )
						self.assertListEqual(_snake_case		,    _snake_case   )

						# With lower casing
						_a							:       str       					=			self.get_tokenizer(do_lower_case=_snake_case   )
						_a							:       int       					=			self.get_rust_tokenizer(do_lower_case=_snake_case   )

						_a							:       Optional[Any]       					=			'''UNwant\u00E9d,running'''

						_a							:       int       					=			tokenizer.tokenize(_snake_case   )
						_a							:       int       					=			rust_tokenizer.tokenize(_snake_case   )
						self.assertListEqual(_snake_case		,    _snake_case   )

						_a							:       List[Any]       					=			tokenizer.encode(_snake_case		,    add_special_tokens=_snake_case   )
						_a							:       Optional[int]       					=			rust_tokenizer.encode(_snake_case		,    add_special_tokens=_snake_case   )
						self.assertListEqual(_snake_case		,    _snake_case   )

						_a							:       Optional[int]       					=			self.get_rust_tokenizer()
						_a							:       Dict       					=			tokenizer.encode(_snake_case   )
						_a							:       List[Any]       					=			rust_tokenizer.encode(_snake_case   )
						self.assertListEqual(_snake_case		,    _snake_case   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       int       					=			BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""   )		,    ["""ah""", """\u535A""", """\u63A8""", """zz"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       List[str]       					=			BasicTokenizer(do_lower_case=_snake_case   )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """   )		,    ["""hello""", """!""", """how""", """are""", """you""", """?"""]   )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""   )		,    ["""hello"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       int       					=			BasicTokenizer(do_lower_case=_snake_case		,    strip_accents=_snake_case   )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """   )		,    ["""hällo""", """!""", """how""", """are""", """you""", """?"""]   )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""   )		,    ["""h\u00E9llo"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       Optional[Any]       					=			BasicTokenizer(do_lower_case=_snake_case		,    strip_accents=_snake_case   )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """   )		,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""]   )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""   )		,    ["""hello"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       Any       					=			BasicTokenizer(do_lower_case=_snake_case   )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """   )		,    ["""hallo""", """!""", """how""", """are""", """you""", """?"""]   )
						self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""   )		,    ["""hello"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       str       					=			BasicTokenizer(do_lower_case=_snake_case   )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """   )		,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       Tuple       					=			BasicTokenizer(do_lower_case=_snake_case		,    strip_accents=_snake_case   )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """   )		,    ["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       Union[str, Any]       					=			BasicTokenizer(do_lower_case=_snake_case		,    strip_accents=_snake_case   )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """   )		,    ["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       Any       					=			BasicTokenizer(do_lower_case=_snake_case		,    never_split=["""[UNK]"""]   )

						self.assertListEqual(
						    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""   )		,    ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       Union[str, Any]       					=			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

						_a							:       Union[str, Any]       					=			{}
						for i, token in enumerate(_snake_case   ):
										_a							:       Optional[int]       					=			i
						_a							:       Any       					=			WordpieceTokenizer(vocab=_snake_case		,    unk_token="""[UNK]"""   )

						self.assertListEqual(tokenizer.tokenize(""""""   )		,    []   )

						self.assertListEqual(tokenizer.tokenize("""unwanted running"""   )		,    ["""un""", """##want""", """##ed""", """runn""", """##ing"""]   )

						self.assertListEqual(tokenizer.tokenize("""unwantedX running"""   )		,    ["""[UNK]""", """runn""", """##ing"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						self.assertTrue(_is_whitespace(""" """   )   )
						self.assertTrue(_is_whitespace("""\t"""   )   )
						self.assertTrue(_is_whitespace("""\r"""   )   )
						self.assertTrue(_is_whitespace("""\n"""   )   )
						self.assertTrue(_is_whitespace("""\u00A0"""   )   )

						self.assertFalse(_is_whitespace("""A"""   )   )
						self.assertFalse(_is_whitespace("""-"""   )   )



		def 						a_       ( self   ):







						'''simple docstring'''


						self.assertTrue(_is_control("""\u0005"""   )   )

						self.assertFalse(_is_control("""A"""   )   )
						self.assertFalse(_is_control(""" """   )   )
						self.assertFalse(_is_control("""\t"""   )   )
						self.assertFalse(_is_control("""\r"""   )   )



		def 						a_       ( self   ):







						'''simple docstring'''


						self.assertTrue(_is_punctuation("""-"""   )   )
						self.assertTrue(_is_punctuation("""$"""   )   )
						self.assertTrue(_is_punctuation("""`"""   )   )
						self.assertTrue(_is_punctuation("""."""   )   )

						self.assertFalse(_is_punctuation("""A"""   )   )
						self.assertFalse(_is_punctuation(""" """   )   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       Optional[int]       					=			self.get_tokenizer()
						_a							:       List[str]       					=			self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(_snake_case   ) for t in ["""Test""", """\xad""", """test"""]]		,    [["""[UNK]"""], [], ["""[UNK]"""]]   )

						self.assertListEqual(
						    [rust_tokenizer.tokenize(_snake_case   ) for t in ["""Test""", """\xad""", """test"""]]		,    [["""[UNK]"""], [], ["""[UNK]"""]]   )



		@slow
		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       str       					=			self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""   )

						_a							:       List[str]       					=			tokenizer.encode("""sequence builders"""		,    add_special_tokens=_snake_case   )
						_a							:       Dict       					=			tokenizer.encode("""multi-sequence build"""		,    add_special_tokens=_snake_case   )

						_a							:       Any       					=			tokenizer.build_inputs_with_special_tokens(_snake_case   )
						_a							:       int       					=			tokenizer.build_inputs_with_special_tokens(_snake_case		,    _snake_case   )

						assert encoded_sentence == [101] + text + [102]
						assert encoded_pair == [101] + text + [102] + text_a + [102]



		def 						a_       ( self   ):







						'''simple docstring'''


						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
														_a							:       Any       					=			self.rust_tokenizer_class.from_pretrained(_snake_case		,    **_snake_case   )

														_a							:       Tuple       					=			f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
														_a							:       str       					=			tokenizer_r.encode_plus(
														    _snake_case		,    return_attention_mask=_snake_case		,    return_token_type_ids=_snake_case		,    return_offsets_mapping=_snake_case		,    add_special_tokens=_snake_case		,    )

														_a							:       List[Any]       					=			tokenizer_r.do_lower_case if hasattr(_snake_case		,    """do_lower_case"""   ) else False
														_a							:       int       					=			(
														    [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''A'''),
														        ((1, 2), ''','''),
														        ((3, 5), '''na'''),
														        ((5, 6), '''##ï'''),
														        ((6, 8), '''##ve'''),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), '''Allen'''),
														        ((21, 23), '''##NL'''),
														        ((23, 24), '''##P'''),
														        ((25, 33), '''sentence'''),
														        ((33, 34), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														    if not do_lower_case
														    else [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''a'''),
														        ((1, 2), ''','''),
														        ((3, 8), '''naive'''),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), '''allen'''),
														        ((21, 23), '''##nl'''),
														        ((23, 24), '''##p'''),
														        ((25, 33), '''sentence'''),
														        ((33, 34), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														)

														self.assertEqual(
														    [e[1] for e in expected_results]		,    tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]   )   )
														self.assertEqual([e[0] for e in expected_results]		,    tokens["""offset_mapping"""]   )



		def 						a_       ( self   ):







						'''simple docstring'''


						_a							:       List[Any]       					=			['''的''', '''人''', '''有''']
						_a							:       Dict       					=			''''''.join(_snake_case   )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
														_a							:       str       					=			True
														_a							:       str       					=			self.tokenizer_class.from_pretrained(_snake_case		,    **_snake_case   )
														_a							:       List[str]       					=			self.rust_tokenizer_class.from_pretrained(_snake_case		,    **_snake_case   )

														_a							:       Union[str, Any]       					=			tokenizer_p.encode(_snake_case		,    add_special_tokens=_snake_case   )
														_a							:       Dict       					=			tokenizer_r.encode(_snake_case		,    add_special_tokens=_snake_case   )

														_a							:       Optional[int]       					=			tokenizer_r.convert_ids_to_tokens(_snake_case   )
														_a							:       Union[str, Any]       					=			tokenizer_p.convert_ids_to_tokens(_snake_case   )

														# it is expected that each Chinese character is not preceded by "##"
														self.assertListEqual(_snake_case		,    _snake_case   )
														self.assertListEqual(_snake_case		,    _snake_case   )

														_a							:       Union[str, Any]       					=			False
														_a							:       Any       					=			self.rust_tokenizer_class.from_pretrained(_snake_case		,    **_snake_case   )
														_a							:       Dict       					=			self.tokenizer_class.from_pretrained(_snake_case		,    **_snake_case   )

														_a							:       int       					=			tokenizer_r.encode(_snake_case		,    add_special_tokens=_snake_case   )
														_a							:       Dict       					=			tokenizer_p.encode(_snake_case		,    add_special_tokens=_snake_case   )

														_a							:       List[Any]       					=			tokenizer_r.convert_ids_to_tokens(_snake_case   )
														_a							:       Tuple       					=			tokenizer_p.convert_ids_to_tokens(_snake_case   )

														# it is expected that only the first Chinese character is not preceded by "##".
														_a							:       Any       					=			[
														    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_snake_case   )
														]
														self.assertListEqual(_snake_case		,    _snake_case   )
														self.assertListEqual(_snake_case		,    _snake_case   )








'''simple docstring'''





def UpperCAmelCase						(					UpperCAmelCase_	):
    _a							:       List[Any]       					=			len(UpperCAmelCase_	)
    for cycle_start in range(0      ,  array_len - 1	):
        _a							:       int       					=			array[cycle_start]

        _a							:       int       					=			cycle_start
        for i in range(cycle_start + 1      ,  UpperCAmelCase_	):
            if array[i] < item:
                pos += 1

        if pos == cycle_start:
            continue

        while item == array[pos]:
            pos += 1

        _a					,   _a							:       List[str]       					=			item, array[pos]
        while pos != cycle_start:
            _a							:       Any       					=			cycle_start
            for i in range(cycle_start + 1      ,  UpperCAmelCase_	):
                if array[i] < item:
                    pos += 1

            while item == array[pos]:
                pos += 1

            _a					,   _a							:       List[Any]       					=			item, array[pos]

    return array


if __name__ == "__main__":
  assert cycle_sort([4, 5, 3, 2, 1]) == [1, 2, 3, 4, 5]
  assert cycle_sort([0, 1, -1_0, 1_5, 2, -2]) == [-1_0, -2, 0, 1, 2, 1_5]


