






'''simple docstring'''







import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class      __UpperCAmelCase					(UpperCamelCase__       ,					unittest.TestCase					):


  """simple docstring"""


  _a     :   List[str]			 =    ElectraTokenizer
  _a     :   List[Any]			 =    ElectraTokenizerFast
  _a     :   List[str]			 =    True
  _a     :   Any			 =    True
  _a     :   List[str]			 =    filter_non_english







  def       _a					( self  ):






         """simple docstring"""


         super().setUp()

         a_	    = [
             '''[UNK]''',
             '''[CLS]''',
             '''[SEP]''',
             '''[PAD]''',
             '''[MASK]''',
             '''want''',
             '''##want''',
             '''##ed''',
             '''wa''',
             '''un''',
             '''runn''',
             '''##ing''',
             ''',''',
             '''low''',
             '''lowest''',
         ]
         a_	    = os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES['vocab_file']  )
         with open(self.vocab_file      ,  'w'      ,  encoding='utf-8'  ) as vocab_writer:
                vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )







  def       _a					( self      ,  UpperCamelCase__  ):






         """simple docstring"""


         a_	    = '''UNwant\u00E9d,running'''
         a_	    = '''unwanted, running'''
         return input_text, output_text







  def       _a					( self  ):






         """simple docstring"""


         a_	    = self.tokenizer_class(self.vocab_file  )

         a_	    = tokenizer.tokenize('UNwant\u00E9d,running'  )
         self.assertListEqual(snake_case_      ,  ['un', '##want', '##ed', ',', 'runn', '##ing']  )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_  )      ,  [9, 6, 7, 12, 10, 11]  )







  def       _a					( self  ):






         """simple docstring"""


         if not self.test_rust_tokenizer:
                return

         a_	    = self.get_tokenizer()
         a_	    = self.get_rust_tokenizer()

         a_	    = '''UNwant\u00E9d,running'''

         a_	    = tokenizer.tokenize(snake_case_  )
         a_	    = rust_tokenizer.tokenize(snake_case_  )
         self.assertListEqual(snake_case_      ,  snake_case_  )

         a_	    = tokenizer.encode(snake_case_      ,  add_special_tokens=snake_case_  )
         a_	    = rust_tokenizer.encode(snake_case_      ,  add_special_tokens=snake_case_  )
         self.assertListEqual(snake_case_      ,  snake_case_  )

         a_	    = self.get_rust_tokenizer()
         a_	    = tokenizer.encode(snake_case_  )
         a_	    = rust_tokenizer.encode(snake_case_  )
         self.assertListEqual(snake_case_      ,  snake_case_  )

         # With lower casing
         a_	    = self.get_tokenizer(do_lower_case=snake_case_  )
         a_	    = self.get_rust_tokenizer(do_lower_case=snake_case_  )

         a_	    = '''UNwant\u00E9d,running'''

         a_	    = tokenizer.tokenize(snake_case_  )
         a_	    = rust_tokenizer.tokenize(snake_case_  )
         self.assertListEqual(snake_case_      ,  snake_case_  )

         a_	    = tokenizer.encode(snake_case_      ,  add_special_tokens=snake_case_  )
         a_	    = rust_tokenizer.encode(snake_case_      ,  add_special_tokens=snake_case_  )
         self.assertListEqual(snake_case_      ,  snake_case_  )

         a_	    = self.get_rust_tokenizer()
         a_	    = tokenizer.encode(snake_case_  )
         a_	    = rust_tokenizer.encode(snake_case_  )
         self.assertListEqual(snake_case_      ,  snake_case_  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )      ,  ['ah', '\u535A', '\u63A8', 'zz']  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = BasicTokenizer(do_lower_case=snake_case_  )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )      ,  ['hello', '!', 'how', 'are', 'you', '?']  )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,  ['hello']  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = BasicTokenizer(do_lower_case=snake_case_      ,  strip_accents=snake_case_  )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,  ['hällo', '!', 'how', 'are', 'you', '?']  )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,  ['h\u00E9llo']  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = BasicTokenizer(do_lower_case=snake_case_      ,  strip_accents=snake_case_  )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,  ['hallo', '!', 'how', 'are', 'you', '?']  )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,  ['hello']  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = BasicTokenizer(do_lower_case=snake_case_  )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,  ['hallo', '!', 'how', 'are', 'you', '?']  )
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,  ['hello']  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = BasicTokenizer(do_lower_case=snake_case_  )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )      ,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = BasicTokenizer(do_lower_case=snake_case_      ,  strip_accents=snake_case_  )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = BasicTokenizer(do_lower_case=snake_case_      ,  strip_accents=snake_case_  )

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = BasicTokenizer(do_lower_case=snake_case_      ,  never_split=['[UNK]']  )

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )      ,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

         a_	    = {}
         for i, token in enumerate(snake_case_  ):
                a_	    = i
         a_	    = WordpieceTokenizer(vocab=snake_case_      ,  unk_token='[UNK]'  )

         self.assertListEqual(tokenizer.tokenize(''  )      ,  []  )

         self.assertListEqual(tokenizer.tokenize('unwanted running'  )      ,  ['un', '##want', '##ed', 'runn', '##ing']  )

         self.assertListEqual(tokenizer.tokenize('unwantedX running'  )      ,  ['[UNK]', 'runn', '##ing']  )







  def       _a					( self  ):






         """simple docstring"""


         self.assertTrue(_is_whitespace(' '  )  )
         self.assertTrue(_is_whitespace('\t'  )  )
         self.assertTrue(_is_whitespace('\r'  )  )
         self.assertTrue(_is_whitespace('\n'  )  )
         self.assertTrue(_is_whitespace('\u00A0'  )  )

         self.assertFalse(_is_whitespace('A'  )  )
         self.assertFalse(_is_whitespace('-'  )  )







  def       _a					( self  ):






         """simple docstring"""


         self.assertTrue(_is_control('\u0005'  )  )

         self.assertFalse(_is_control('A'  )  )
         self.assertFalse(_is_control(' '  )  )
         self.assertFalse(_is_control('\t'  )  )
         self.assertFalse(_is_control('\r'  )  )







  def       _a					( self  ):






         """simple docstring"""


         self.assertTrue(_is_punctuation('-'  )  )
         self.assertTrue(_is_punctuation('$'  )  )
         self.assertTrue(_is_punctuation('`'  )  )
         self.assertTrue(_is_punctuation('.'  )  )

         self.assertFalse(_is_punctuation('A'  )  )
         self.assertFalse(_is_punctuation(' '  )  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = self.get_tokenizer()
         a_	    = self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(snake_case_  ) for t in ['Test', '\xad', 'test']]      ,  [['[UNK]'], [], ['[UNK]']]  )

         self.assertListEqual(
             [rust_tokenizer.tokenize(snake_case_  ) for t in ['Test', '\xad', 'test']]      ,  [['[UNK]'], [], ['[UNK]']]  )







  @slow
  def       _a					( self  ):






         """simple docstring"""


         a_	    = self.tokenizer_class.from_pretrained('google/electra-base-discriminator'  )

         a_	    = tokenizer.encode('sequence builders'      ,  add_special_tokens=snake_case_  )
         a_	    = tokenizer.encode('multi-sequence build'      ,  add_special_tokens=snake_case_  )

         a_	    = tokenizer.build_inputs_with_special_tokens(snake_case_  )
         a_	    = tokenizer.build_inputs_with_special_tokens(snake_case_      ,  snake_case_  )

         assert encoded_sentence == [101] + text + [102]
         assert encoded_pair == [101] + text + [102] + text_a + [102]







  def       _a					( self  ):






         """simple docstring"""


         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
                       a_	    = self.rust_tokenizer_class.from_pretrained(snake_case_      ,  **snake_case_  )

                       a_	    = f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                       a_	    = tokenizer_r.encode_plus(
                           snake_case_      ,  return_attention_mask=snake_case_      ,  return_token_type_ids=snake_case_      ,  return_offsets_mapping=snake_case_      ,  add_special_tokens=snake_case_      ,  )

                       a_	    = tokenizer_r.do_lower_case if hasattr(snake_case_      ,  'do_lower_case'  ) else False
                       a_	    = (
                           [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), '''A'''),
                               ((1, 2), ''','''),
                               ((3, 5), '''na'''),
                               ((5, 6), '''##ï'''),
                               ((6, 8), '''##ve'''),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), '''Allen'''),
                               ((21, 23), '''##NL'''),
                               ((23, 24), '''##P'''),
                               ((25, 33), '''sentence'''),
                               ((33, 34), '''.'''),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                           if not do_lower_case
                           else [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), '''a'''),
                               ((1, 2), ''','''),
                               ((3, 8), '''naive'''),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), '''allen'''),
                               ((21, 23), '''##nl'''),
                               ((23, 24), '''##p'''),
                               ((25, 33), '''sentence'''),
                               ((33, 34), '''.'''),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                       )

                       self.assertEqual(
                           [e[1] for e in expected_results]      ,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
                       self.assertEqual([e[0] for e in expected_results]      ,  tokens['offset_mapping']  )







  def       _a					( self  ):






         """simple docstring"""


         a_	    = ['''的''', '''人''', '''有''']
         a_	    = ''''''.join(snake_case_  )
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
                       a_	    = True
                       a_	    = self.tokenizer_class.from_pretrained(snake_case_      ,  **snake_case_  )
                       a_	    = self.rust_tokenizer_class.from_pretrained(snake_case_      ,  **snake_case_  )

                       a_	    = tokenizer_p.encode(snake_case_      ,  add_special_tokens=snake_case_  )
                       a_	    = tokenizer_r.encode(snake_case_      ,  add_special_tokens=snake_case_  )

                       a_	    = tokenizer_r.convert_ids_to_tokens(snake_case_  )
                       a_	    = tokenizer_p.convert_ids_to_tokens(snake_case_  )

                       # it is expected that each Chinese character is not preceded by "##"
                       self.assertListEqual(snake_case_      ,  snake_case_  )
                       self.assertListEqual(snake_case_      ,  snake_case_  )

                       a_	    = False
                       a_	    = self.rust_tokenizer_class.from_pretrained(snake_case_      ,  **snake_case_  )
                       a_	    = self.tokenizer_class.from_pretrained(snake_case_      ,  **snake_case_  )

                       a_	    = tokenizer_r.encode(snake_case_      ,  add_special_tokens=snake_case_  )
                       a_	    = tokenizer_p.encode(snake_case_      ,  add_special_tokens=snake_case_  )

                       a_	    = tokenizer_r.convert_ids_to_tokens(snake_case_  )
                       a_	    = tokenizer_p.convert_ids_to_tokens(snake_case_  )

                       # it is expected that only the first Chinese character is not preceded by "##".
                       a_	    = [
                           f'##{token}' if idx != 0 else token for idx, token in enumerate(snake_case_  )
                       ]
                       self.assertListEqual(snake_case_      ,  snake_case_  )
                       self.assertListEqual(snake_case_      ,  snake_case_  )












'''simple docstring'''







def 			__UpperCamelCase				(			lowercase_					:     int      ,					lowercase_					:     int      ):
       """simple docstring"""




       return (input_a, input_a).count(0      ) % 2


def 			__UpperCamelCase				(			):
       """simple docstring"""




       assert xor_gate(0      ,					0      ) == 0
       assert xor_gate(0      ,					1      ) == 1
       assert xor_gate(1      ,					0      ) == 1
       assert xor_gate(1      ,					1      ) == 0


if __name__ == "__main__":
  print(xor_gate(0, 0))
  print(xor_gate(0, 1))




