


'''simple docstring'''





import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 							UpperCAmelCase ( _snake_case	,   unittest.TestCase	):
 UpperCAmelCase				=				ElectraTokenizer
 UpperCAmelCase				=				ElectraTokenizerFast
 UpperCAmelCase				=				True
 UpperCAmelCase				=				True
 UpperCAmelCase				=				filter_non_english


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Any   ):
    super().setUp()

    UpperCAmelCase__  :int            =			[
        "[UNK]",
        "[CLS]",
        "[SEP]",
        "[PAD]",
        "[MASK]",
        "want",
        "##want",
        "##ed",
        "wa",
        "un",
        "runn",
        "##ing",
        ",",
        "low",
        "lowest",
    ]
    UpperCAmelCase__  :int            =			os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['''vocab_file''']   )
    with open(self.vocab_file ,					'''w''' ,					encoding='''utf-8'''   ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Optional[int] ,					__lowerCamelCase       :  Optional[int]   ):
    UpperCAmelCase__  :int            =			"UNwant\u00E9d,running"
    UpperCAmelCase__  :Dict            =			"unwanted, running"
    return input_text, output_text


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Any   ):
    UpperCAmelCase__  :Union[str, Any]            =			self.tokenizer_class(self.vocab_file   )

    UpperCAmelCase__  :int            =			tokenizer.tokenize('''UNwant\u00E9d,running'''   )
    self.assertListEqual(__lowerCamelCase ,					['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']   )
    self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase   ) ,					[9, 6, 7, 1_2, 1_0, 1_1]   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Optional[int]   ):






    if not self.test_rust_tokenizer:
       return

    UpperCAmelCase__  :Any            =			self.get_tokenizer()
    UpperCAmelCase__  :Dict            =			self.get_rust_tokenizer()

    UpperCAmelCase__  :str            =			"UNwant\u00E9d,running"

    UpperCAmelCase__  :Tuple            =			tokenizer.tokenize(__lowerCamelCase   )
    UpperCAmelCase__  :Optional[Any]            =			rust_tokenizer.tokenize(__lowerCamelCase   )
    self.assertListEqual(__lowerCamelCase ,					__lowerCamelCase   )

    UpperCAmelCase__  :str            =			tokenizer.encode(__lowerCamelCase ,					add_special_tokens=__lowerCamelCase   )
    UpperCAmelCase__  :str            =			rust_tokenizer.encode(__lowerCamelCase ,					add_special_tokens=__lowerCamelCase   )
    self.assertListEqual(__lowerCamelCase ,					__lowerCamelCase   )

    UpperCAmelCase__  :Optional[Any]            =			self.get_rust_tokenizer()
    UpperCAmelCase__  :List[Any]            =			tokenizer.encode(__lowerCamelCase   )
    UpperCAmelCase__  :Optional[int]            =			rust_tokenizer.encode(__lowerCamelCase   )
    self.assertListEqual(__lowerCamelCase ,					__lowerCamelCase   )

    # With lower casing
    UpperCAmelCase__  :Union[str, Any]            =			self.get_tokenizer(do_lower_case=__lowerCamelCase   )
    UpperCAmelCase__  :Dict            =			self.get_rust_tokenizer(do_lower_case=__lowerCamelCase   )

    UpperCAmelCase__  :Any            =			"UNwant\u00E9d,running"

    UpperCAmelCase__  :Tuple            =			tokenizer.tokenize(__lowerCamelCase   )
    UpperCAmelCase__  :Optional[Any]            =			rust_tokenizer.tokenize(__lowerCamelCase   )
    self.assertListEqual(__lowerCamelCase ,					__lowerCamelCase   )

    UpperCAmelCase__  :str            =			tokenizer.encode(__lowerCamelCase ,					add_special_tokens=__lowerCamelCase   )
    UpperCAmelCase__  :Optional[Any]            =			rust_tokenizer.encode(__lowerCamelCase ,					add_special_tokens=__lowerCamelCase   )
    self.assertListEqual(__lowerCamelCase ,					__lowerCamelCase   )

    UpperCAmelCase__  :List[Any]            =			self.get_rust_tokenizer()
    UpperCAmelCase__  :List[str]            =			tokenizer.encode(__lowerCamelCase   )
    UpperCAmelCase__  :Any            =			rust_tokenizer.encode(__lowerCamelCase   )
    self.assertListEqual(__lowerCamelCase ,					__lowerCamelCase   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Optional[Any]   ):
    UpperCAmelCase__  :Any            =			BasicTokenizer()

    self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''   ) ,					['''ah''', '''\u535A''', '''\u63A8''', '''zz''']   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  List[str]   ):
    UpperCAmelCase__  :str            =			BasicTokenizer(do_lower_case=__lowerCamelCase   )

    self.assertListEqual(
        tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   ) ,					['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
    self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   ) ,					['''hello''']   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  str   ):
    UpperCAmelCase__  :int            =			BasicTokenizer(do_lower_case=__lowerCamelCase ,					strip_accents=__lowerCamelCase   )

    self.assertListEqual(
        tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   ) ,					['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
    self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   ) ,					['''h\u00E9llo''']   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  List[Any]   ):
    UpperCAmelCase__  :Dict            =			BasicTokenizer(do_lower_case=__lowerCamelCase ,					strip_accents=__lowerCamelCase   )

    self.assertListEqual(
        tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   ) ,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
    self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   ) ,					['''hello''']   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Union[str, Any]   ):
    UpperCAmelCase__  :List[Any]            =			BasicTokenizer(do_lower_case=__lowerCamelCase   )

    self.assertListEqual(
        tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   ) ,					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']   )
    self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''   ) ,					['''hello''']   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  List[Any]   ):
    UpperCAmelCase__  :Union[str, Any]            =			BasicTokenizer(do_lower_case=__lowerCamelCase   )

    self.assertListEqual(
        tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''   ) ,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  int   ):
    UpperCAmelCase__  :Optional[Any]            =			BasicTokenizer(do_lower_case=__lowerCamelCase ,					strip_accents=__lowerCamelCase   )

    self.assertListEqual(
        tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   ) ,					['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Optional[Any]   ):
    UpperCAmelCase__  :str            =			BasicTokenizer(do_lower_case=__lowerCamelCase ,					strip_accents=__lowerCamelCase   )

    self.assertListEqual(
        tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''   ) ,					['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Tuple   ):
    UpperCAmelCase__  :int            =			BasicTokenizer(do_lower_case=__lowerCamelCase ,					never_split=['''[UNK]''']   )

    self.assertListEqual(
        tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''   ) ,					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Optional[Any]   ):
    UpperCAmelCase__  :Union[str, Any]            =			["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

    UpperCAmelCase__  :Tuple            =			{}
    for i, token in enumerate(__lowerCamelCase   ):
       UpperCAmelCase__  :str            =			i
    UpperCAmelCase__  :Any            =			WordpieceTokenizer(vocab=__lowerCamelCase ,					unk_token='''[UNK]'''   )

    self.assertListEqual(tokenizer.tokenize(''''''   ) ,					[]   )

    self.assertListEqual(tokenizer.tokenize('''unwanted running'''   ) ,					['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']   )

    self.assertListEqual(tokenizer.tokenize('''unwantedX running'''   ) ,					['''[UNK]''', '''runn''', '''##ing''']   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Dict   ):
    self.assertTrue(_is_whitespace(''' '''   )   )
    self.assertTrue(_is_whitespace('''\t'''   )   )
    self.assertTrue(_is_whitespace('''\r'''   )   )
    self.assertTrue(_is_whitespace('''\n'''   )   )
    self.assertTrue(_is_whitespace('''\u00A0'''   )   )

    self.assertFalse(_is_whitespace('''A'''   )   )
    self.assertFalse(_is_whitespace('''-'''   )   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Any   ):
    self.assertTrue(_is_control('''\u0005'''   )   )

    self.assertFalse(_is_control('''A'''   )   )
    self.assertFalse(_is_control(''' '''   )   )
    self.assertFalse(_is_control('''\t'''   )   )
    self.assertFalse(_is_control('''\r'''   )   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Union[str, Any]   ):
    self.assertTrue(_is_punctuation('''-'''   )   )
    self.assertTrue(_is_punctuation('''$'''   )   )
    self.assertTrue(_is_punctuation('''`'''   )   )
    self.assertTrue(_is_punctuation('''.'''   )   )

    self.assertFalse(_is_punctuation('''A'''   )   )
    self.assertFalse(_is_punctuation(''' '''   )   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  str   ):
    UpperCAmelCase__  :Tuple            =			self.get_tokenizer()
    UpperCAmelCase__  :Dict            =			self.get_rust_tokenizer()

    # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
    self.assertListEqual([tokenizer.tokenize(__lowerCamelCase   ) for t in ['''Test''', '''\xad''', '''test''']] ,					[['''[UNK]'''], [], ['''[UNK]''']]   )

    self.assertListEqual(
        [rust_tokenizer.tokenize(__lowerCamelCase   ) for t in ['''Test''', '''\xad''', '''test''']] ,					[['''[UNK]'''], [], ['''[UNK]''']]   )


 @slow
 def 					__SCREAMING_SNAKE_CASE				(			self       :  Optional[int]   ):
    UpperCAmelCase__  :List[str]            =			self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''   )

    UpperCAmelCase__  :Dict            =			tokenizer.encode('''sequence builders''' ,					add_special_tokens=__lowerCamelCase   )
    UpperCAmelCase__  :Optional[Any]            =			tokenizer.encode('''multi-sequence build''' ,					add_special_tokens=__lowerCamelCase   )

    UpperCAmelCase__  :str            =			tokenizer.build_inputs_with_special_tokens(__lowerCamelCase   )
    UpperCAmelCase__  :Tuple            =			tokenizer.build_inputs_with_special_tokens(__lowerCamelCase ,					__lowerCamelCase   )

    assert encoded_sentence == [1_0_1] + text + [1_0_2]
    assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Optional[Any]   ):






    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
          UpperCAmelCase__  :Any            =			self.rust_tokenizer_class.from_pretrained(__lowerCamelCase ,					**__lowerCamelCase   )

          UpperCAmelCase__  :Optional[Any]            =			f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
          UpperCAmelCase__  :Union[str, Any]            =			tokenizer_r.encode_plus(
              __lowerCamelCase ,					return_attention_mask=__lowerCamelCase ,					return_token_type_ids=__lowerCamelCase ,					return_offsets_mapping=__lowerCamelCase ,					add_special_tokens=__lowerCamelCase ,					)

          UpperCAmelCase__  :Optional[int]            =			tokenizer_r.do_lower_case if hasattr(__lowerCamelCase ,					'''do_lower_case'''   ) else False
          UpperCAmelCase__  :Tuple            =			(
              [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), "A"),
                  ((1, 2), ","),
                  ((3, 5), "na"),
                  ((5, 6), "##ï"),
                  ((6, 8), "##ve"),
                  ((9, 1_5), tokenizer_r.mask_token),
                  ((1_6, 2_1), "Allen"),
                  ((2_1, 2_3), "##NL"),
                  ((2_3, 2_4), "##P"),
                  ((2_5, 3_3), "sentence"),
                  ((3_3, 3_4), "."),
                  ((0, 0), tokenizer_r.sep_token),
              ]
              if not do_lower_case
              else [
                  ((0, 0), tokenizer_r.cls_token),
                  ((0, 1), "a"),
                  ((1, 2), ","),
                  ((3, 8), "naive"),
                  ((9, 1_5), tokenizer_r.mask_token),
                  ((1_6, 2_1), "allen"),
                  ((2_1, 2_3), "##nl"),
                  ((2_3, 2_4), "##p"),
                  ((2_5, 3_3), "sentence"),
                  ((3_3, 3_4), "."),
                  ((0, 0), tokenizer_r.sep_token),
              ]
          )

          self.assertEqual(
              [e[1] for e in expected_results] ,					tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']   )   )
          self.assertEqual([e[0] for e in expected_results] ,					tokens['''offset_mapping''']   )





 def 					__SCREAMING_SNAKE_CASE				(			self       :  Tuple   ):
    UpperCAmelCase__  :Union[str, Any]            =			["的", "人", "有"]
    UpperCAmelCase__  :Optional[int]            =			"".join(__lowerCamelCase   )
    for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
       with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
          UpperCAmelCase__  :List[Any]            =			True
          UpperCAmelCase__  :str            =			self.tokenizer_class.from_pretrained(__lowerCamelCase ,					**__lowerCamelCase   )
          UpperCAmelCase__  :Tuple            =			self.rust_tokenizer_class.from_pretrained(__lowerCamelCase ,					**__lowerCamelCase   )

          UpperCAmelCase__  :Union[str, Any]            =			tokenizer_p.encode(__lowerCamelCase ,					add_special_tokens=__lowerCamelCase   )
          UpperCAmelCase__  :Optional[Any]            =			tokenizer_r.encode(__lowerCamelCase ,					add_special_tokens=__lowerCamelCase   )

          UpperCAmelCase__  :Any            =			tokenizer_r.convert_ids_to_tokens(__lowerCamelCase   )
          UpperCAmelCase__  :Any            =			tokenizer_p.convert_ids_to_tokens(__lowerCamelCase   )

          # it is expected that each Chinese character is not preceded by "##"
          self.assertListEqual(__lowerCamelCase ,					__lowerCamelCase   )
          self.assertListEqual(__lowerCamelCase ,					__lowerCamelCase   )

          UpperCAmelCase__  :Tuple            =			False
          UpperCAmelCase__  :Tuple            =			self.rust_tokenizer_class.from_pretrained(__lowerCamelCase ,					**__lowerCamelCase   )
          UpperCAmelCase__  :List[str]            =			self.tokenizer_class.from_pretrained(__lowerCamelCase ,					**__lowerCamelCase   )

          UpperCAmelCase__  :Optional[Any]            =			tokenizer_r.encode(__lowerCamelCase ,					add_special_tokens=__lowerCamelCase   )
          UpperCAmelCase__  :Optional[Any]            =			tokenizer_p.encode(__lowerCamelCase ,					add_special_tokens=__lowerCamelCase   )

          UpperCAmelCase__  :List[Any]            =			tokenizer_r.convert_ids_to_tokens(__lowerCamelCase   )
          UpperCAmelCase__  :Union[str, Any]            =			tokenizer_p.convert_ids_to_tokens(__lowerCamelCase   )

          # it is expected that only the first Chinese character is not preceded by "##".
          UpperCAmelCase__  :Optional[int]            =			[
              f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__lowerCamelCase   )
          ]
          self.assertListEqual(__lowerCamelCase ,					__lowerCamelCase   )
          self.assertListEqual(__lowerCamelCase ,					__lowerCamelCase   )







'''simple docstring'''





from typing import List, Optional, Tuple, Union

from . import logging
from .import_utils import is_torch_available, is_torch_version


if is_torch_available():
    import torch

__lowerCamelCase       				=	logging.get_logger(__name__)  # pylint: disable=invalid-name

try:
    from torch._dynamo import allow_in_graph as maybe_allow_in_graph
except (ImportError, ModuleNotFoundError):


    def 					a__    (				cls	:			Optional[int]	):
       return cls


def 					a__    (				UpperCamelCase_	:			Union[Tuple, List],      UpperCamelCase_	:			Optional[Union[List["torch.Generator"], "torch.Generator"]] = None,      UpperCamelCase_	:			Optional["torch.device"] = None,      UpperCamelCase_	:			Optional["torch.dtype"] = None,      UpperCamelCase_	:			Optional["torch.layout"] = None,      ):
   UpperCAmelCase__  :Union[str, Any]            =			device
   UpperCAmelCase__  :List[Any]            =			shape[0]

   UpperCAmelCase__  :Tuple            =			layout or torch.strided
   UpperCAmelCase__  :Union[str, Any]            =			device or torch.device('''cpu'''	)

   if generator is not None:
      UpperCAmelCase__  :str            =			generator.device.type if not isinstance(UpperCamelCase_,      UpperCamelCase_	) else generator[0].device.type
      if gen_device_type != device.type and gen_device_type == "cpu":
         UpperCAmelCase__  :Tuple            =			'''cpu'''
         if device != "mps":
            logger.info(
                F'''The passed generator was created on \'cpu\' even though a tensor on {device} was expected.'''
                F''' Tensors will be created on \'cpu\' and then moved to {device}. Note that one can probably'''
                F''' slighly speed up this function by passing a generator that was created on the {device} device.'''	)
      elif gen_device_type != device.type and gen_device_type == "cuda":
         raise ValueError(F'''Cannot generate a {device} tensor from a generator of type {gen_device_type}.'''	)

   if isinstance(UpperCamelCase_,      UpperCamelCase_	):
      UpperCAmelCase__  :List[str]            =			(1,) + shape[1:]
      UpperCAmelCase__  :Optional[int]            =			[
          torch.randn(UpperCamelCase_,      generator=generator[i],      device=UpperCamelCase_,      dtype=UpperCamelCase_,      layout=UpperCamelCase_	)
          for i in range(UpperCamelCase_	)
      ]
      UpperCAmelCase__  :int            =			torch.cat(UpperCamelCase_,      dim=0	).to(UpperCamelCase_	)
   else:
      UpperCAmelCase__  :Optional[int]            =			torch.randn(UpperCamelCase_,      generator=UpperCamelCase_,      device=UpperCamelCase_,      dtype=UpperCamelCase_,      layout=UpperCamelCase_	).to(UpperCamelCase_	)

   return latents


def 					a__    (				UpperCamelCase_	:			Dict	):
   if is_torch_version('''<''',      '''2.0.0'''	) or not hasattr(UpperCamelCase_,      '''_dynamo'''	):
      return False
   return isinstance(UpperCamelCase_,      torch._dynamo.eval_frame.OptimizedModule	)



