'''simple docstring'''






import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class lowercase_					(		A_						,						unittest.TestCase ):







   """simple docstring"""

   __lowerCAmelCase    						=      ElectraTokenizer
   __lowerCAmelCase    						=      ElectraTokenizerFast
   __lowerCAmelCase    						=      True
   __lowerCAmelCase    						=      True
   __lowerCAmelCase    						=      filter_non_english
   def   __UpperCAmelCase      (			self			: Optional[int]    )     -> Tuple:
        super().setUp()

        _A								=     [
            '[UNK]',
            '[CLS]',
            '[SEP]',
            '[PAD]',
            '[MASK]',
            'want',
            '##want',
            '##ed',
            'wa',
            'un',
            'runn',
            '##ing',
            ',',
            'low',
            'lowest',
        ]
        _A								=     os.path.join(self.tmpdirname,   VOCAB_FILES_NAMES['vocab_file']    )
        with open(self.vocab_file,   'w',   encoding='utf-8'    ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )
   def   __UpperCAmelCase      (			self			: List[str],   UpperCamelCase__			: Tuple    )     -> Any:
        _A								=     'UNwant\u00E9d,running'
        _A								=     'unwanted, running'
        return input_text, output_text
   def   __UpperCAmelCase      (			self			: Optional[Any]    )     -> str:
        _A								=     self.tokenizer_class(self.vocab_file    )

        _A								=     tokenizer.tokenize('UNwant\u00E9d,running'    )
        self.assertListEqual(UpperCamelCase__,   ['un', '##want', '##ed', ',', 'runn', '##ing']    )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__    ),   [9, 6, 7, 12, 10, 11]    )
   def   __UpperCAmelCase      (			self			: Optional[Any]    )     -> str:
        if not self.test_rust_tokenizer:
             return

        _A								=     self.get_tokenizer()
        _A								=     self.get_rust_tokenizer()

        _A								=     'UNwant\u00E9d,running'

        _A								=     tokenizer.tokenize(UpperCamelCase__    )
        _A								=     rust_tokenizer.tokenize(UpperCamelCase__    )
        self.assertListEqual(UpperCamelCase__,   UpperCamelCase__    )

        _A								=     tokenizer.encode(UpperCamelCase__,   add_special_tokens=UpperCamelCase__    )
        _A								=     rust_tokenizer.encode(UpperCamelCase__,   add_special_tokens=UpperCamelCase__    )
        self.assertListEqual(UpperCamelCase__,   UpperCamelCase__    )

        _A								=     self.get_rust_tokenizer()
        _A								=     tokenizer.encode(UpperCamelCase__    )
        _A								=     rust_tokenizer.encode(UpperCamelCase__    )
        self.assertListEqual(UpperCamelCase__,   UpperCamelCase__    )

        # With lower casing
        _A								=     self.get_tokenizer(do_lower_case=UpperCamelCase__    )
        _A								=     self.get_rust_tokenizer(do_lower_case=UpperCamelCase__    )

        _A								=     'UNwant\u00E9d,running'

        _A								=     tokenizer.tokenize(UpperCamelCase__    )
        _A								=     rust_tokenizer.tokenize(UpperCamelCase__    )
        self.assertListEqual(UpperCamelCase__,   UpperCamelCase__    )

        _A								=     tokenizer.encode(UpperCamelCase__,   add_special_tokens=UpperCamelCase__    )
        _A								=     rust_tokenizer.encode(UpperCamelCase__,   add_special_tokens=UpperCamelCase__    )
        self.assertListEqual(UpperCamelCase__,   UpperCamelCase__    )

        _A								=     self.get_rust_tokenizer()
        _A								=     tokenizer.encode(UpperCamelCase__    )
        _A								=     rust_tokenizer.encode(UpperCamelCase__    )
        self.assertListEqual(UpperCamelCase__,   UpperCamelCase__    )
   def   __UpperCAmelCase      (			self			: Tuple    )     -> str:
        _A								=     BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'    ),   ['ah', '\u535A', '\u63A8', 'zz']    )
   def   __UpperCAmelCase      (			self			: Optional[int]    )     -> Optional[int]:
        _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    ),   ['hello', '!', 'how', 'are', 'you', '?']    )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    ),   ['hello']    )
   def   __UpperCAmelCase      (			self			: List[str]    )     -> str:
        _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__,   strip_accents=UpperCamelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    ),   ['hällo', '!', 'how', 'are', 'you', '?']    )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    ),   ['h\u00E9llo']    )
   def   __UpperCAmelCase      (			self			: Optional[Any]    )     -> Dict:
        _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__,   strip_accents=UpperCamelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    ),   ['hallo', '!', 'how', 'are', 'you', '?']    )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    ),   ['hello']    )
   def   __UpperCAmelCase      (			self			: int    )     -> int:
        _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    ),   ['hallo', '!', 'how', 'are', 'you', '?']    )
        self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    ),   ['hello']    )
   def   __UpperCAmelCase      (			self			: Any    )     -> List[Any]:
        _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    ),   ['HeLLo', '!', 'how', 'Are', 'yoU', '?']    )
   def   __UpperCAmelCase      (			self			: Optional[Any]    )     -> Union[str, Any]:
        _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__,   strip_accents=UpperCamelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    ),   ['HäLLo', '!', 'how', 'Are', 'yoU', '?']    )
   def   __UpperCAmelCase      (			self			: Optional[Any]    )     -> str:
        _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__,   strip_accents=UpperCamelCase__    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    ),   ['HaLLo', '!', 'how', 'Are', 'yoU', '?']    )
   def   __UpperCAmelCase      (			self			: Dict    )     -> Dict:
        _A								=     BasicTokenizer(do_lower_case=UpperCamelCase__,   never_split=['[UNK]']    )

        self.assertListEqual(
            tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'    ),   ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']    )
   def   __UpperCAmelCase      (			self			: int    )     -> List[str]:
        _A								=     ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

        _A								=     {}
        for i, token in enumerate(UpperCamelCase__    ):
             _A								=     i
        _A								=     WordpieceTokenizer(vocab=UpperCamelCase__,   unk_token='[UNK]'    )

        self.assertListEqual(tokenizer.tokenize(''    ),   []    )

        self.assertListEqual(tokenizer.tokenize('unwanted running'    ),   ['un', '##want', '##ed', 'runn', '##ing']    )

        self.assertListEqual(tokenizer.tokenize('unwantedX running'    ),   ['[UNK]', 'runn', '##ing']    )
   def   __UpperCAmelCase      (			self			: List[str]    )     -> Optional[Any]:
        self.assertTrue(_is_whitespace(' '    )    )
        self.assertTrue(_is_whitespace('\t'    )    )
        self.assertTrue(_is_whitespace('\r'    )    )
        self.assertTrue(_is_whitespace('\n'    )    )
        self.assertTrue(_is_whitespace('\u00A0'    )    )

        self.assertFalse(_is_whitespace('A'    )    )
        self.assertFalse(_is_whitespace('-'    )    )
   def   __UpperCAmelCase      (			self			: List[str]    )     -> List[Any]:
        self.assertTrue(_is_control('\u0005'    )    )

        self.assertFalse(_is_control('A'    )    )
        self.assertFalse(_is_control(' '    )    )
        self.assertFalse(_is_control('\t'    )    )
        self.assertFalse(_is_control('\r'    )    )
   def   __UpperCAmelCase      (			self			: str    )     -> Dict:
        self.assertTrue(_is_punctuation('-'    )    )
        self.assertTrue(_is_punctuation('$'    )    )
        self.assertTrue(_is_punctuation('`'    )    )
        self.assertTrue(_is_punctuation('.'    )    )

        self.assertFalse(_is_punctuation('A'    )    )
        self.assertFalse(_is_punctuation(' '    )    )
   def   __UpperCAmelCase      (			self			: List[str]    )     -> Optional[Any]:
        _A								=     self.get_tokenizer()
        _A								=     self.get_rust_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(UpperCamelCase__    ) for t in ['Test', '\xad', 'test']],   [['[UNK]'], [], ['[UNK]']]    )

        self.assertListEqual(
            [rust_tokenizer.tokenize(UpperCamelCase__    ) for t in ['Test', '\xad', 'test']],   [['[UNK]'], [], ['[UNK]']]    )
   @slow
   def   __UpperCAmelCase      (			self			: Tuple    )     -> Tuple:
        _A								=     self.tokenizer_class.from_pretrained('google/electra-base-discriminator'    )

        _A								=     tokenizer.encode('sequence builders',   add_special_tokens=UpperCamelCase__    )
        _A								=     tokenizer.encode('multi-sequence build',   add_special_tokens=UpperCamelCase__    )

        _A								=     tokenizer.build_inputs_with_special_tokens(UpperCamelCase__    )
        _A								=     tokenizer.build_inputs_with_special_tokens(UpperCamelCase__,   UpperCamelCase__    )

        assert encoded_sentence == [1_01] + text + [1_02]
        assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]
   def   __UpperCAmelCase      (			self			: int    )     -> Union[str, Any]:
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'    ):
                  _A								=     self.rust_tokenizer_class.from_pretrained(UpperCamelCase__,   **UpperCamelCase__    )

                  _A								=     f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                  _A								=     tokenizer_r.encode_plus(
                      UpperCamelCase__,   return_attention_mask=UpperCamelCase__,   return_token_type_ids=UpperCamelCase__,   return_offsets_mapping=UpperCamelCase__,   add_special_tokens=UpperCamelCase__,   )

                  _A								=     tokenizer_r.do_lower_case if hasattr(UpperCamelCase__,   'do_lower_case'    ) else False
                  _A								=     (
                      [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), 'A'),
                          ((1, 2), ','),
                          ((3, 5), 'na'),
                          ((5, 6), '##ï'),
                          ((6, 8), '##ve'),
                          ((9, 15), tokenizer_r.mask_token),
                          ((16, 21), 'Allen'),
                          ((21, 23), '##NL'),
                          ((23, 24), '##P'),
                          ((25, 33), 'sentence'),
                          ((33, 34), '.'),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                      if not do_lower_case
                      else [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), 'a'),
                          ((1, 2), ','),
                          ((3, 8), 'naive'),
                          ((9, 15), tokenizer_r.mask_token),
                          ((16, 21), 'allen'),
                          ((21, 23), '##nl'),
                          ((23, 24), '##p'),
                          ((25, 33), 'sentence'),
                          ((33, 34), '.'),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                  )

                  self.assertEqual(
                      [e[1] for e in expected_results],   tokenizer_r.convert_ids_to_tokens(tokens['input_ids']    )    )
                  self.assertEqual([e[0] for e in expected_results],   tokens['offset_mapping']    )

   def   __UpperCAmelCase      (			self			: Dict    )     -> str:
        _A								=     ['的', '人', '有']
        _A								=     ''.join(UpperCamelCase__    )
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'    ):
                  _A								=     True
                  _A								=     self.tokenizer_class.from_pretrained(UpperCamelCase__,   **UpperCamelCase__    )
                  _A								=     self.rust_tokenizer_class.from_pretrained(UpperCamelCase__,   **UpperCamelCase__    )

                  _A								=     tokenizer_p.encode(UpperCamelCase__,   add_special_tokens=UpperCamelCase__    )
                  _A								=     tokenizer_r.encode(UpperCamelCase__,   add_special_tokens=UpperCamelCase__    )

                  _A								=     tokenizer_r.convert_ids_to_tokens(UpperCamelCase__    )
                  _A								=     tokenizer_p.convert_ids_to_tokens(UpperCamelCase__    )

                  # it is expected that each Chinese character is not preceded by "##"
                  self.assertListEqual(UpperCamelCase__,   UpperCamelCase__    )
                  self.assertListEqual(UpperCamelCase__,   UpperCamelCase__    )

                  _A								=     False
                  _A								=     self.rust_tokenizer_class.from_pretrained(UpperCamelCase__,   **UpperCamelCase__    )
                  _A								=     self.tokenizer_class.from_pretrained(UpperCamelCase__,   **UpperCamelCase__    )

                  _A								=     tokenizer_r.encode(UpperCamelCase__,   add_special_tokens=UpperCamelCase__    )
                  _A								=     tokenizer_p.encode(UpperCamelCase__,   add_special_tokens=UpperCamelCase__    )

                  _A								=     tokenizer_r.convert_ids_to_tokens(UpperCamelCase__    )
                  _A								=     tokenizer_p.convert_ids_to_tokens(UpperCamelCase__    )

                  # it is expected that only the first Chinese character is not preceded by "##".
                  _A								=     [
                      f'##{token}' if idx != 0 else token for idx, token in enumerate(UpperCamelCase__    )
                  ]
                  self.assertListEqual(UpperCamelCase__,   UpperCamelCase__    )
                  self.assertListEqual(UpperCamelCase__,   UpperCamelCase__    )




'''simple docstring'''






import torch

from accelerate import Accelerator, DistributedType







class lowercase_					:







   """simple docstring"""

   def __enter__(			self			: Optional[int]    )     -> Union[str, Any]:
        if self.enabled:
             _A								=     self.model.no_sync()
             self.model_sync_obj.__enter__()

        return self
   def __exit__(			self			: Dict,   UpperCamelCase__			: Any,   UpperCamelCase__			: int,   UpperCamelCase__			: Any    )     -> str:
        if self.enabled:
             # Average all models on exit
             self._sync_and_avg_model_params()
             self.model_sync_obj.__exit__(UpperCamelCase__,   UpperCamelCase__,   UpperCamelCase__    )
   def __init__(			self			: Optional[int],   UpperCamelCase__			: Accelerator,   UpperCamelCase__			: torch.nn.Module,   UpperCamelCase__			: int,   UpperCamelCase__			: bool = True    )     -> str:
        if accelerator.distributed_type not in [
            DistributedType.NO,
            DistributedType.MULTI_CPU,
            DistributedType.MULTI_GPU,
        ]:
             raise NotImplementedError('LocalSGD is supported only for CPUs and GPUs (no DeepSpeed or MegatronLM)'    )
        _A								=     enabled and accelerator.distributed_type != DistributedType.NO
        _A								=     0
        if self.enabled:
             _A								=     accelerator
             _A								=     model
             _A								=     local_sgd_steps
   def   __UpperCAmelCase      (			self			: Any    )     -> int:
        self.num_steps += 1
        if not self.enabled:
             return

        if self.num_steps % self.local_sgd_steps == 0:
             self._sync_and_avg_model_params()

   def   __UpperCAmelCase      (			self			: Dict    )     -> int:
        self.accelerator.wait_for_everyone()
        with self.accelerator.autocast():
             for param in self.model.parameters():
                  _A								=     self.accelerator.reduce(param.data,   reduction='mean'    )



