


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 						A__      (					snake_case__ ,      unittest.TestCase			):







    """simple docstring"""

    __magic_name__    						=	ElectraTokenizer
    __magic_name__    						=	ElectraTokenizerFast
    __magic_name__    						=	True
    __magic_name__    						=	True
    __magic_name__    						=	filter_non_english
    def 	a_    ( self					):
         super().setUp()

         snake_case					=       [
             '''[UNK]''',
             '''[CLS]''',
             '''[SEP]''',
             '''[PAD]''',
             '''[MASK]''',
             '''want''',
             '''##want''',
             '''##ed''',
             '''wa''',
             '''un''',
             '''runn''',
             '''##ing''',
             ''',''',
             '''low''',
             '''lowest''',
         ]
         snake_case					=       os.path.join(self.tmpdirname    ,      VOCAB_FILES_NAMES['''vocab_file''']					)
         with open(self.vocab_file    ,      '''w'''    ,      encoding='''utf-8'''					) as vocab_writer:
              vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]					)					)
    def 	a_    ( self    ,      __snake_case					):
         snake_case					=       '''UNwant\u00E9d,running'''
         snake_case					=       '''unwanted, running'''
         return input_text, output_text
    def 	a_    ( self					):
         snake_case					=       self.tokenizer_class(self.vocab_file					)

         snake_case					=       tokenizer.tokenize('''UNwant\u00E9d,running'''					)
         self.assertListEqual(__snake_case    ,      ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']					)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case					)    ,      [9, 6, 7, 1_2, 1_0, 1_1]					)
    def 	a_    ( self					):
         if not self.test_rust_tokenizer:
              return

         snake_case					=       self.get_tokenizer()
         snake_case					=       self.get_rust_tokenizer()

         snake_case					=       '''UNwant\u00E9d,running'''

         snake_case					=       tokenizer.tokenize(__snake_case					)
         snake_case					=       rust_tokenizer.tokenize(__snake_case					)
         self.assertListEqual(__snake_case    ,      __snake_case					)

         snake_case					=       tokenizer.encode(__snake_case    ,      add_special_tokens=__snake_case					)
         snake_case					=       rust_tokenizer.encode(__snake_case    ,      add_special_tokens=__snake_case					)
         self.assertListEqual(__snake_case    ,      __snake_case					)

         snake_case					=       self.get_rust_tokenizer()
         snake_case					=       tokenizer.encode(__snake_case					)
         snake_case					=       rust_tokenizer.encode(__snake_case					)
         self.assertListEqual(__snake_case    ,      __snake_case					)

         # With lower casing
         snake_case					=       self.get_tokenizer(do_lower_case=__snake_case					)
         snake_case					=       self.get_rust_tokenizer(do_lower_case=__snake_case					)

         snake_case					=       '''UNwant\u00E9d,running'''

         snake_case					=       tokenizer.tokenize(__snake_case					)
         snake_case					=       rust_tokenizer.tokenize(__snake_case					)
         self.assertListEqual(__snake_case    ,      __snake_case					)

         snake_case					=       tokenizer.encode(__snake_case    ,      add_special_tokens=__snake_case					)
         snake_case					=       rust_tokenizer.encode(__snake_case    ,      add_special_tokens=__snake_case					)
         self.assertListEqual(__snake_case    ,      __snake_case					)

         snake_case					=       self.get_rust_tokenizer()
         snake_case					=       tokenizer.encode(__snake_case					)
         snake_case					=       rust_tokenizer.encode(__snake_case					)
         self.assertListEqual(__snake_case    ,      __snake_case					)
    def 	a_    ( self					):
         snake_case					=       BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''					)    ,      ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']					)
    def 	a_    ( self					):
         snake_case					=       BasicTokenizer(do_lower_case=__snake_case					)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)    ,      ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)    ,      ['''hello''']					)
    def 	a_    ( self					):
         snake_case					=       BasicTokenizer(do_lower_case=__snake_case    ,      strip_accents=__snake_case					)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)    ,      ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)    ,      ['''h\u00E9llo''']					)
    def 	a_    ( self					):
         snake_case					=       BasicTokenizer(do_lower_case=__snake_case    ,      strip_accents=__snake_case					)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)    ,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)    ,      ['''hello''']					)
    def 	a_    ( self					):
         snake_case					=       BasicTokenizer(do_lower_case=__snake_case					)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)    ,      ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']					)
         self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''					)    ,      ['''hello''']					)
    def 	a_    ( self					):
         snake_case					=       BasicTokenizer(do_lower_case=__snake_case					)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''					)    ,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
    def 	a_    ( self					):
         snake_case					=       BasicTokenizer(do_lower_case=__snake_case    ,      strip_accents=__snake_case					)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)    ,      ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
    def 	a_    ( self					):
         snake_case					=       BasicTokenizer(do_lower_case=__snake_case    ,      strip_accents=__snake_case					)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''					)    ,      ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']					)
    def 	a_    ( self					):
         snake_case					=       BasicTokenizer(do_lower_case=__snake_case    ,      never_split=['''[UNK]''']					)

         self.assertListEqual(
             tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''					)    ,      ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']					)
    def 	a_    ( self					):
         snake_case					=       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

         snake_case					=       {}
         for i, token in enumerate(__snake_case					):
              snake_case					=       i
         snake_case					=       WordpieceTokenizer(vocab=__snake_case    ,      unk_token='''[UNK]'''					)

         self.assertListEqual(tokenizer.tokenize(''''''					)    ,      []					)

         self.assertListEqual(tokenizer.tokenize('''unwanted running'''					)    ,      ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']					)

         self.assertListEqual(tokenizer.tokenize('''unwantedX running'''					)    ,      ['''[UNK]''', '''runn''', '''##ing''']					)
    def 	a_    ( self					):
         self.assertTrue(_is_whitespace(''' '''					)					)
         self.assertTrue(_is_whitespace('''\t'''					)					)
         self.assertTrue(_is_whitespace('''\r'''					)					)
         self.assertTrue(_is_whitespace('''\n'''					)					)
         self.assertTrue(_is_whitespace('''\u00A0'''					)					)

         self.assertFalse(_is_whitespace('''A'''					)					)
         self.assertFalse(_is_whitespace('''-'''					)					)
    def 	a_    ( self					):
         self.assertTrue(_is_control('''\u0005'''					)					)

         self.assertFalse(_is_control('''A'''					)					)
         self.assertFalse(_is_control(''' '''					)					)
         self.assertFalse(_is_control('''\t'''					)					)
         self.assertFalse(_is_control('''\r'''					)					)
    def 	a_    ( self					):
         self.assertTrue(_is_punctuation('''-'''					)					)
         self.assertTrue(_is_punctuation('''$'''					)					)
         self.assertTrue(_is_punctuation('''`'''					)					)
         self.assertTrue(_is_punctuation('''.'''					)					)

         self.assertFalse(_is_punctuation('''A'''					)					)
         self.assertFalse(_is_punctuation(''' '''					)					)
    def 	a_    ( self					):
         snake_case					=       self.get_tokenizer()
         snake_case					=       self.get_rust_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(__snake_case					) for t in ['''Test''', '''\xad''', '''test''']]    ,      [['''[UNK]'''], [], ['''[UNK]''']]					)

         self.assertListEqual(
             [rust_tokenizer.tokenize(__snake_case					) for t in ['''Test''', '''\xad''', '''test''']]    ,      [['''[UNK]'''], [], ['''[UNK]''']]					)
    @slow
    def 	a_    ( self					):
         snake_case					=       self.tokenizer_class.from_pretrained('''google/electra-base-discriminator'''					)

         snake_case					=       tokenizer.encode('''sequence builders'''    ,      add_special_tokens=__snake_case					)
         snake_case					=       tokenizer.encode('''multi-sequence build'''    ,      add_special_tokens=__snake_case					)

         snake_case					=       tokenizer.build_inputs_with_special_tokens(__snake_case					)
         snake_case					=       tokenizer.build_inputs_with_special_tokens(__snake_case    ,      __snake_case					)

         assert encoded_sentence == [1_0_1] + text + [1_0_2]
         assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
    def 	a_    ( self					):
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                   snake_case					=       self.rust_tokenizer_class.from_pretrained(__snake_case    ,      **__snake_case					)

                   snake_case					=       F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                   snake_case					=       tokenizer_r.encode_plus(
                       __snake_case    ,      return_attention_mask=__snake_case    ,      return_token_type_ids=__snake_case    ,      return_offsets_mapping=__snake_case    ,      add_special_tokens=__snake_case    ,      )

                   snake_case					=       tokenizer_r.do_lower_case if hasattr(__snake_case    ,      '''do_lower_case'''					) else False
                   snake_case					=       (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''A'''),
                           ((1, 2), ''','''),
                           ((3, 5), '''na'''),
                           ((5, 6), '''##ï'''),
                           ((6, 8), '''##ve'''),
                           ((9, 1_5), tokenizer_r.mask_token),
                           ((1_6, 2_1), '''Allen'''),
                           ((2_1, 2_3), '''##NL'''),
                           ((2_3, 2_4), '''##P'''),
                           ((2_5, 3_3), '''sentence'''),
                           ((3_3, 3_4), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), '''a'''),
                           ((1, 2), ''','''),
                           ((3, 8), '''naive'''),
                           ((9, 1_5), tokenizer_r.mask_token),
                           ((1_6, 2_1), '''allen'''),
                           ((2_1, 2_3), '''##nl'''),
                           ((2_3, 2_4), '''##p'''),
                           ((2_5, 3_3), '''sentence'''),
                           ((3_3, 3_4), '''.'''),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]    ,      tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']					)					)
                   self.assertEqual([e[0] for e in expected_results]    ,      tokens['''offset_mapping''']					)




    def 	a_    ( self					):
         snake_case					=       ['''的''', '''人''', '''有''']
         snake_case					=       ''''''.join(__snake_case					)
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
                   snake_case					=       True
                   snake_case					=       self.tokenizer_class.from_pretrained(__snake_case    ,      **__snake_case					)
                   snake_case					=       self.rust_tokenizer_class.from_pretrained(__snake_case    ,      **__snake_case					)

                   snake_case					=       tokenizer_p.encode(__snake_case    ,      add_special_tokens=__snake_case					)
                   snake_case					=       tokenizer_r.encode(__snake_case    ,      add_special_tokens=__snake_case					)

                   snake_case					=       tokenizer_r.convert_ids_to_tokens(__snake_case					)
                   snake_case					=       tokenizer_p.convert_ids_to_tokens(__snake_case					)

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(__snake_case    ,      __snake_case					)
                   self.assertListEqual(__snake_case    ,      __snake_case					)

                   snake_case					=       False
                   snake_case					=       self.rust_tokenizer_class.from_pretrained(__snake_case    ,      **__snake_case					)
                   snake_case					=       self.tokenizer_class.from_pretrained(__snake_case    ,      **__snake_case					)

                   snake_case					=       tokenizer_r.encode(__snake_case    ,      add_special_tokens=__snake_case					)
                   snake_case					=       tokenizer_p.encode(__snake_case    ,      add_special_tokens=__snake_case					)

                   snake_case					=       tokenizer_r.convert_ids_to_tokens(__snake_case					)
                   snake_case					=       tokenizer_p.convert_ids_to_tokens(__snake_case					)

                   # it is expected that only the first Chinese character is not preceded by "##".
                   snake_case					=       [
                       F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__snake_case					)
                   ]
                   self.assertListEqual(__snake_case    ,      __snake_case					)
                   self.assertListEqual(__snake_case    ,      __snake_case					)








from manim import *

class 						A__      (					snake_case__			):







    """simple docstring"""

    def 	a_    ( self					):
         snake_case					=       Rectangle(height=0.5    ,      width=0.5					)
         snake_case					=       Rectangle(height=0.46    ,      width=0.46					).set_stroke(width=0					)
         snake_case					=       Rectangle(height=0.25    ,      width=0.25					)

         snake_case					=       [mem.copy() for i in range(6					)]
         snake_case					=       [mem.copy() for i in range(6					)]
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       VGroup(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       Text('''CPU'''    ,      font_size=2_4					)
         snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
         cpu.move_to([-2.5, -0.5, 0]					)
         self.add(__snake_case					)

         snake_case					=       [mem.copy() for i in range(4					)]
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       Text('''GPU'''    ,      font_size=2_4					)
         snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
         gpu.move_to([-1, -1, 0]					)
         self.add(__snake_case					)

         snake_case					=       [mem.copy() for i in range(6					)]
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)

         snake_case					=       Text('''Model'''    ,      font_size=2_4					)
         snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
         model.move_to([3, -1.0, 0]					)
         self.add(__snake_case					)

         snake_case					=       []
         snake_case					=       []

         for i, rect in enumerate(__snake_case					):
              rect.set_stroke(__snake_case					)

              snake_case					=       Rectangle(height=0.46 / 4    ,      width=0.46 / 3					).set_stroke(width=0.0					).set_fill(__snake_case    ,      opacity=0.7					)

              if i == 0:
                   cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT					)    ,      buff=0.02    ,      direction=__snake_case					)
                   cpu_target.set_x(cpu_target.get_x() + 0.1					)
              elif i == 3:
                   cpu_target.next_to(model_cpu_arr[0]    ,      direction=__snake_case    ,      buff=0.0					)
              else:
                   cpu_target.next_to(model_cpu_arr[i - 1]    ,      direction=__snake_case    ,      buff=0.0					)
              self.add(__snake_case					)
              model_cpu_arr.append(__snake_case					)

         self.add(*__snake_case    ,      *__snake_case					)

         snake_case					=       [meta_mem.copy() for i in range(6					)]
         snake_case					=       [meta_mem.copy() for i in range(6					)]
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       VGroup(*__snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       VGroup(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0					)
         snake_case					=       Text('''Disk'''    ,      font_size=2_4					)
         snake_case					=       Group(__snake_case    ,      __snake_case					).arrange(__snake_case    ,      buff=0.5    ,      aligned_edge=__snake_case					)
         disk.move_to([-4.0, -1.25, 0]					)
         self.add(__snake_case    ,      __snake_case					)

         snake_case					=       []

         for i in range(6					):
              snake_case					=       fill.copy().set_fill(__snake_case    ,      opacity=0.8					)
              target.move_to(disk_left_col_base[i]					).scale(0.5					)
              cpu_disk_arr.append(__snake_case					)

         self.add(*__snake_case					)

         snake_case					=       Square(side_length=2.2					)
         key.move_to([-5, 2, 0]					)

         snake_case					=       MarkupText(
             F'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''    ,      font_size=1_8    ,      )

         key_text.move_to([-5, 2.4, 0]					)

         self.add(__snake_case    ,      __snake_case					)

         snake_case					=       MarkupText(
             F'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''    ,      font_size=1_8    ,      )

         blue_text.next_to(__snake_case    ,      DOWN * 2.4    ,      aligned_edge=key_text.get_left()					)
         self.add(__snake_case					)

         snake_case					=       MarkupText(F'''The offloaded weights are all sent to the CPU.'''    ,      font_size=2_4					)
         step_a.move_to([2, 2, 0]					)

         self.play(Write(__snake_case    ,      run_time=3					)					)

         for i in range(6					):
              snake_case					=       cpu_disk_arr[i]
              snake_case					=       rect.copy().set_fill(__snake_case    ,      opacity=0.8					).scale(2.0					)
              cpa.generate_target()
              cpa.target.move_to(model_base[i]					)

              if i == 0:
                   rect.set_fill(__snake_case    ,      opacity=0.8					)
                   rect.generate_target()
                   rect.target.move_to(cpu_left_col_base[0]					).scale(2.0					)

                   self.remove(
                       *__snake_case    ,      *__snake_case    ,      )

              else:
                   rect.generate_target()
                   rect.target.move_to(cpu_left_col_base[i]					).scale(2.0					)
              self.play(MoveToTarget(__snake_case					)    ,      MoveToTarget(__snake_case					)    ,      model_base[i].animate.set_stroke(__snake_case					)					)
         self.play(FadeOut(__snake_case					)					)

         snake_case					=       MarkupText(
             F'''Finally, hooks are added to each weight in the model\nto transfer the weights from CPU to GPU\n\t\tand back when needed.'''    ,      font_size=2_4    ,      )
         step_a.move_to([2, 2, 0]					)

         self.play(Write(__snake_case    ,      run_time=3					)					)

         snake_case					=       []
         snake_case					=       []
         for i in range(6					):
              snake_case					=       Arrow(start=__snake_case    ,      end=__snake_case    ,      color=__snake_case    ,      buff=0.5					)
              a.next_to(model_base[i].get_left()    ,      __snake_case    ,      buff=0.2					)
              arrows.append(__snake_case					)
              animations.append(Write(__snake_case					)					)
         self.play(*__snake_case					)
         self.wait()




