




'''simple docstring'''






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 	SCREAMING_SNAKE_CASE_		(							lowerCamelCase_		,  unittest.TestCase					):
 __snake_case						:  Dict       =   ElectraTokenizer
 __snake_case						:  Any       =   ElectraTokenizerFast
 __snake_case						:  Tuple       =   True
 __snake_case						:  List[Any]       =   True
 __snake_case						:  int       =   filter_non_english



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        super().setUp()

        lowercase	:		List[Any]    =   [
            '''[UNK]''',
            '''[CLS]''',
            '''[SEP]''',
            '''[PAD]''',
            '''[MASK]''',
            '''want''',
            '''##want''',
            '''##ed''',
            '''wa''',
            '''un''',
            '''runn''',
            '''##ing''',
            ''',''',
            '''low''',
            '''lowest''',
        ]
        lowercase	:		Optional[Any]    =   os.path.join(self.tmpdirname ,		VOCAB_FILES_NAMES['''vocab_file''']			)
        with open(self.vocab_file ,		'''w''' ,		encoding='''utf-8'''			) as vocab_writer:
               vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)



 def  SCREAMING_SNAKE_CASE(	self ,		A			):






        '''simple docstring'''




        lowercase	:		List[Any]    =   '''UNwant\u00E9d,running'''
        lowercase	:		List[str]    =   '''unwanted, running'''
        return input_text, output_text



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		Any    =   self.tokenizer_class(self.vocab_file			)

        lowercase	:		Any    =   tokenizer.tokenize('''UNwant\u00E9d,running'''			)
        self.assertListEqual(A ,		['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']			)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(A			) ,		[9, 6, 7, 12, 10, 11]			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        if not self.test_rust_tokenizer:
               return

        lowercase	:		int    =   self.get_tokenizer()
        lowercase	:		Any    =   self.get_rust_tokenizer()

        lowercase	:		str    =   '''UNwant\u00E9d,running'''

        lowercase	:		Dict    =   tokenizer.tokenize(A			)
        lowercase	:		str    =   rust_tokenizer.tokenize(A			)
        self.assertListEqual(A ,		A			)

        lowercase	:		Union[str, Any]    =   tokenizer.encode(A ,		add_special_tokens=A			)
        lowercase	:		Optional[int]    =   rust_tokenizer.encode(A ,		add_special_tokens=A			)
        self.assertListEqual(A ,		A			)

        lowercase	:		List[str]    =   self.get_rust_tokenizer()
        lowercase	:		int    =   tokenizer.encode(A			)
        lowercase	:		Dict    =   rust_tokenizer.encode(A			)
        self.assertListEqual(A ,		A			)

        # With lower casing
        lowercase	:		Union[str, Any]    =   self.get_tokenizer(do_lower_case=A			)
        lowercase	:		Optional[int]    =   self.get_rust_tokenizer(do_lower_case=A			)

        lowercase	:		Optional[int]    =   '''UNwant\u00E9d,running'''

        lowercase	:		str    =   tokenizer.tokenize(A			)
        lowercase	:		Tuple    =   rust_tokenizer.tokenize(A			)
        self.assertListEqual(A ,		A			)

        lowercase	:		Union[str, Any]    =   tokenizer.encode(A ,		add_special_tokens=A			)
        lowercase	:		int    =   rust_tokenizer.encode(A ,		add_special_tokens=A			)
        self.assertListEqual(A ,		A			)

        lowercase	:		Any    =   self.get_rust_tokenizer()
        lowercase	:		List[str]    =   tokenizer.encode(A			)
        lowercase	:		Any    =   rust_tokenizer.encode(A			)
        self.assertListEqual(A ,		A			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		Tuple    =   BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''			) ,		['''ah''', '''\u535A''', '''\u63A8''', '''zz''']			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		List[Any]    =   BasicTokenizer(do_lower_case=A			)

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			) ,		['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			) ,		['''hello''']			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		int    =   BasicTokenizer(do_lower_case=A ,		strip_accents=A			)

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			) ,		['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			) ,		['''h\u00E9llo''']			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		Any    =   BasicTokenizer(do_lower_case=A ,		strip_accents=A			)

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			) ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			) ,		['''hello''']			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		Optional[Any]    =   BasicTokenizer(do_lower_case=A			)

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			) ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']			)
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			) ,		['''hello''']			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		Optional[Any]    =   BasicTokenizer(do_lower_case=A			)

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''			) ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		Any    =   BasicTokenizer(do_lower_case=A ,		strip_accents=A			)

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			) ,		['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		int    =   BasicTokenizer(do_lower_case=A ,		strip_accents=A			)

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''			) ,		['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		Optional[Any]    =   BasicTokenizer(do_lower_case=A ,		never_split=['''[UNK]''']			)

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''			) ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		Union[str, Any]    =   ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

        lowercase	:		Any    =   {}
        for i, token in enumerate(A			):
               lowercase	:		List[str]    =   i
        lowercase	:		Optional[int]    =   WordpieceTokenizer(vocab=A ,		unk_token='''[UNK]'''			)

        self.assertListEqual(tokenizer.tokenize(''''''			) ,		[]			)

        self.assertListEqual(tokenizer.tokenize('''unwanted running'''			) ,		['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']			)

        self.assertListEqual(tokenizer.tokenize('''unwantedX running'''			) ,		['''[UNK]''', '''runn''', '''##ing''']			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        self.assertTrue(_is_whitespace(''' '''			)			)
        self.assertTrue(_is_whitespace('''\t'''			)			)
        self.assertTrue(_is_whitespace('''\r'''			)			)
        self.assertTrue(_is_whitespace('''\n'''			)			)
        self.assertTrue(_is_whitespace('''\u00A0'''			)			)

        self.assertFalse(_is_whitespace('''A'''			)			)
        self.assertFalse(_is_whitespace('''-'''			)			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        self.assertTrue(_is_control('''\u0005'''			)			)

        self.assertFalse(_is_control('''A'''			)			)
        self.assertFalse(_is_control(''' '''			)			)
        self.assertFalse(_is_control('''\t'''			)			)
        self.assertFalse(_is_control('''\r'''			)			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        self.assertTrue(_is_punctuation('''-'''			)			)
        self.assertTrue(_is_punctuation('''$'''			)			)
        self.assertTrue(_is_punctuation('''`'''			)			)
        self.assertTrue(_is_punctuation('''.'''			)			)

        self.assertFalse(_is_punctuation('''A'''			)			)
        self.assertFalse(_is_punctuation(''' '''			)			)



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		int    =   self.get_tokenizer()
        lowercase	:		List[str]    =   self.get_rust_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(A			) for t in ['''Test''', '''\xad''', '''test''']] ,		[['''[UNK]'''], [], ['''[UNK]''']]			)

        self.assertListEqual(
            [rust_tokenizer.tokenize(A			) for t in ['''Test''', '''\xad''', '''test''']] ,		[['''[UNK]'''], [], ['''[UNK]''']]			)



 @slow
 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		Any    =   self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''			)

        lowercase	:		Tuple    =   tokenizer.encode('''sequence builders''' ,		add_special_tokens=A			)
        lowercase	:		Union[str, Any]    =   tokenizer.encode('''multi-sequence build''' ,		add_special_tokens=A			)

        lowercase	:		List[str]    =   tokenizer.build_inputs_with_special_tokens(A			)
        lowercase	:		Optional[int]    =   tokenizer.build_inputs_with_special_tokens(A ,		A			)

        assert encoded_sentence == [101] + text + [102]
        assert encoded_pair == [101] + text + [102] + text_a + [102]



 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                      lowercase	:		str    =   self.rust_tokenizer_class.from_pretrained(A ,		**A			)

                      lowercase	:		List[str]    =   f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                      lowercase	:		str    =   tokenizer_r.encode_plus(
                          A ,		return_attention_mask=A ,		return_token_type_ids=A ,		return_offsets_mapping=A ,		add_special_tokens=A ,		)

                      lowercase	:		List[str]    =   tokenizer_r.do_lower_case if hasattr(A ,		'''do_lower_case'''			) else False
                      lowercase	:		str    =   (
                          [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), '''A'''),
                              ((1, 2), ''','''),
                              ((3, 5), '''na'''),
                              ((5, 6), '''##ï'''),
                              ((6, 8), '''##ve'''),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), '''Allen'''),
                              ((21, 23), '''##NL'''),
                              ((23, 24), '''##P'''),
                              ((25, 33), '''sentence'''),
                              ((33, 34), '''.'''),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                          if not do_lower_case
                          else [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), '''a'''),
                              ((1, 2), ''','''),
                              ((3, 8), '''naive'''),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), '''allen'''),
                              ((21, 23), '''##nl'''),
                              ((23, 24), '''##p'''),
                              ((25, 33), '''sentence'''),
                              ((33, 34), '''.'''),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                      )

                      self.assertEqual(
                          [e[1] for e in expected_results] ,		tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']			)			)
                      self.assertEqual([e[0] for e in expected_results] ,		tokens['''offset_mapping''']			)





 def  SCREAMING_SNAKE_CASE(	self			):






        '''simple docstring'''




        lowercase	:		Dict    =   ['''的''', '''人''', '''有''']
        lowercase	:		str    =   ''''''.join(A			)
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
                      lowercase	:		Optional[Any]    =   True
                      lowercase	:		List[Any]    =   self.tokenizer_class.from_pretrained(A ,		**A			)
                      lowercase	:		str    =   self.rust_tokenizer_class.from_pretrained(A ,		**A			)

                      lowercase	:		Dict    =   tokenizer_p.encode(A ,		add_special_tokens=A			)
                      lowercase	:		int    =   tokenizer_r.encode(A ,		add_special_tokens=A			)

                      lowercase	:		str    =   tokenizer_r.convert_ids_to_tokens(A			)
                      lowercase	:		Union[str, Any]    =   tokenizer_p.convert_ids_to_tokens(A			)

                      # it is expected that each Chinese character is not preceded by "##"
                      self.assertListEqual(A ,		A			)
                      self.assertListEqual(A ,		A			)

                      lowercase	:		Optional[Any]    =   False
                      lowercase	:		Optional[Any]    =   self.rust_tokenizer_class.from_pretrained(A ,		**A			)
                      lowercase	:		Optional[int]    =   self.tokenizer_class.from_pretrained(A ,		**A			)

                      lowercase	:		int    =   tokenizer_r.encode(A ,		add_special_tokens=A			)
                      lowercase	:		Union[str, Any]    =   tokenizer_p.encode(A ,		add_special_tokens=A			)

                      lowercase	:		str    =   tokenizer_r.convert_ids_to_tokens(A			)
                      lowercase	:		Union[str, Any]    =   tokenizer_p.convert_ids_to_tokens(A			)

                      # it is expected that only the first Chinese character is not preceded by "##".
                      lowercase	:		Any    =   [
                          f"""##{token}""" if idx != 0 else token for idx, token in enumerate(A			)
                      ]
                      self.assertListEqual(A ,		A			)
                      self.assertListEqual(A ,		A			)










'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_torch_available


lowercase						:   int      						=							{
    '''configuration_xlm_prophetnet''': ['''XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''XLMProphetNetConfig'''],
}

try:
      if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase						:   Any      						=							['''XLMProphetNetTokenizer''']

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      lowercase						:   Tuple      						=							[
          '''XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
          '''XLMProphetNetDecoder''',
          '''XLMProphetNetEncoder''',
          '''XLMProphetNetForCausalLM''',
          '''XLMProphetNetForConditionalGeneration''',
          '''XLMProphetNetModel''',
          '''XLMProphetNetPreTrainedModel''',
      ]


if TYPE_CHECKING:
      from .configuration_xlm_prophetnet import XLM_PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMProphetNetConfig

      try:
            if not is_sentencepiece_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_xlm_prophetnet import XLMProphetNetTokenizer

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_xlm_prophetnet import (
                XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST,
                XLMProphetNetDecoder,
                XLMProphetNetEncoder,
                XLMProphetNetForCausalLM,
                XLMProphetNetForConditionalGeneration,
                XLMProphetNetModel,
                XLMProphetNetPreTrainedModel,
            )

else:
      import sys

      lowercase						:   Optional[Any]      						=							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




