


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class    __lowercase  (       UpperCamelCase__      ,     unittest.TestCase							):

							"""simple docstring"""


							SCREAMING_SNAKE_CASE   :					Optional[Any]        =						ElectraTokenizer
							SCREAMING_SNAKE_CASE   :					Optional[int]        =						ElectraTokenizerFast
							SCREAMING_SNAKE_CASE   :					str        =						True
							SCREAMING_SNAKE_CASE   :					str        =						True
							SCREAMING_SNAKE_CASE   :					Optional[Any]        =						filter_non_english





							def 		__magic_name__			(    self	)->	Tuple:
											super().setUp()

											_SCREAMING_SNAKE_CASE      = [
											    '''[UNK]''',
											    '''[CLS]''',
											    '''[SEP]''',
											    '''[PAD]''',
											    '''[MASK]''',
											    '''want''',
											    '''##want''',
											    '''##ed''',
											    '''wa''',
											    '''un''',
											    '''runn''',
											    '''##ing''',
											    ''',''',
											    '''low''',
											    '''lowest''',
											]
											_SCREAMING_SNAKE_CASE      = os.path.join(self.tmpdirname				, VOCAB_FILES_NAMES['vocab_file']	)
											with open(self.vocab_file				, 'w'				, encoding='utf-8'	) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]	)	)





							def 		__magic_name__			(    self				, A_	)->	Tuple:
											_SCREAMING_SNAKE_CASE      = '''UNwant\u00E9d,running'''
											_SCREAMING_SNAKE_CASE      = '''unwanted, running'''
											return input_text, output_text





							def 		__magic_name__			(    self	)->	Tuple:
											_SCREAMING_SNAKE_CASE      = self.tokenizer_class(self.vocab_file	)

											_SCREAMING_SNAKE_CASE      = tokenizer.tokenize('UNwant\u00E9d,running'	)
											self.assertListEqual(snake_case_				, ['un', '##want', '##ed', ',', 'runn', '##ing']	)
											self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_	)				, [9, 6, 7, 12, 10, 11]	)





							def 		__magic_name__			(    self	)->	Tuple:
											if not self.test_rust_tokenizer:
															return

											_SCREAMING_SNAKE_CASE      = self.get_tokenizer()
											_SCREAMING_SNAKE_CASE      = self.get_rust_tokenizer()

											_SCREAMING_SNAKE_CASE      = '''UNwant\u00E9d,running'''

											_SCREAMING_SNAKE_CASE      = tokenizer.tokenize(snake_case_	)
											_SCREAMING_SNAKE_CASE      = rust_tokenizer.tokenize(snake_case_	)
											self.assertListEqual(snake_case_				, snake_case_	)

											_SCREAMING_SNAKE_CASE      = tokenizer.encode(snake_case_				, add_special_tokens=snake_case_	)
											_SCREAMING_SNAKE_CASE      = rust_tokenizer.encode(snake_case_				, add_special_tokens=snake_case_	)
											self.assertListEqual(snake_case_				, snake_case_	)

											_SCREAMING_SNAKE_CASE      = self.get_rust_tokenizer()
											_SCREAMING_SNAKE_CASE      = tokenizer.encode(snake_case_	)
											_SCREAMING_SNAKE_CASE      = rust_tokenizer.encode(snake_case_	)
											self.assertListEqual(snake_case_				, snake_case_	)

											# With lower casing
											_SCREAMING_SNAKE_CASE      = self.get_tokenizer(do_lower_case=snake_case_	)
											_SCREAMING_SNAKE_CASE      = self.get_rust_tokenizer(do_lower_case=snake_case_	)

											_SCREAMING_SNAKE_CASE      = '''UNwant\u00E9d,running'''

											_SCREAMING_SNAKE_CASE      = tokenizer.tokenize(snake_case_	)
											_SCREAMING_SNAKE_CASE      = rust_tokenizer.tokenize(snake_case_	)
											self.assertListEqual(snake_case_				, snake_case_	)

											_SCREAMING_SNAKE_CASE      = tokenizer.encode(snake_case_				, add_special_tokens=snake_case_	)
											_SCREAMING_SNAKE_CASE      = rust_tokenizer.encode(snake_case_				, add_special_tokens=snake_case_	)
											self.assertListEqual(snake_case_				, snake_case_	)

											_SCREAMING_SNAKE_CASE      = self.get_rust_tokenizer()
											_SCREAMING_SNAKE_CASE      = tokenizer.encode(snake_case_	)
											_SCREAMING_SNAKE_CASE      = rust_tokenizer.encode(snake_case_	)
											self.assertListEqual(snake_case_				, snake_case_	)





							def 		__magic_name__			(    self	)->	Dict:
											_SCREAMING_SNAKE_CASE      = BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'	)				, ['ah', '\u535A', '\u63A8', 'zz']	)





							def 		__magic_name__			(    self	)->	Tuple:
											_SCREAMING_SNAKE_CASE      = BasicTokenizer(do_lower_case=snake_case_	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '	)				, ['hello', '!', 'how', 'are', 'you', '?']	)
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)				, ['hello']	)





							def 		__magic_name__			(    self	)->	List[str]:
											_SCREAMING_SNAKE_CASE      = BasicTokenizer(do_lower_case=snake_case_				, strip_accents=snake_case_	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)				, ['hällo', '!', 'how', 'are', 'you', '?']	)
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)				, ['h\u00E9llo']	)





							def 		__magic_name__			(    self	)->	List[Any]:
											_SCREAMING_SNAKE_CASE      = BasicTokenizer(do_lower_case=snake_case_				, strip_accents=snake_case_	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)				, ['hallo', '!', 'how', 'are', 'you', '?']	)
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)				, ['hello']	)





							def 		__magic_name__			(    self	)->	Any:
											_SCREAMING_SNAKE_CASE      = BasicTokenizer(do_lower_case=snake_case_	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)				, ['hallo', '!', 'how', 'are', 'you', '?']	)
											self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)				, ['hello']	)





							def 		__magic_name__			(    self	)->	Tuple:
											_SCREAMING_SNAKE_CASE      = BasicTokenizer(do_lower_case=snake_case_	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '	)				, ['HeLLo', '!', 'how', 'Are', 'yoU', '?']	)





							def 		__magic_name__			(    self	)->	Union[str, Any]:
											_SCREAMING_SNAKE_CASE      = BasicTokenizer(do_lower_case=snake_case_				, strip_accents=snake_case_	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)				, ['HäLLo', '!', 'how', 'Are', 'yoU', '?']	)





							def 		__magic_name__			(    self	)->	Any:
											_SCREAMING_SNAKE_CASE      = BasicTokenizer(do_lower_case=snake_case_				, strip_accents=snake_case_	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '	)				, ['HaLLo', '!', 'how', 'Are', 'yoU', '?']	)





							def 		__magic_name__			(    self	)->	List[str]:
											_SCREAMING_SNAKE_CASE      = BasicTokenizer(do_lower_case=snake_case_				, never_split=['[UNK]']	)

											self.assertListEqual(
											    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'	)				, ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']	)





							def 		__magic_name__			(    self	)->	str:
											_SCREAMING_SNAKE_CASE      = ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

											_SCREAMING_SNAKE_CASE      = {}
											for i, token in enumerate(snake_case_	):
															_SCREAMING_SNAKE_CASE      = i
											_SCREAMING_SNAKE_CASE      = WordpieceTokenizer(vocab=snake_case_				, unk_token='[UNK]'	)

											self.assertListEqual(tokenizer.tokenize(''	)				, []	)

											self.assertListEqual(tokenizer.tokenize('unwanted running'	)				, ['un', '##want', '##ed', 'runn', '##ing']	)

											self.assertListEqual(tokenizer.tokenize('unwantedX running'	)				, ['[UNK]', 'runn', '##ing']	)





							def 		__magic_name__			(    self	)->	str:
											self.assertTrue(_is_whitespace(' '	)	)
											self.assertTrue(_is_whitespace('\t'	)	)
											self.assertTrue(_is_whitespace('\r'	)	)
											self.assertTrue(_is_whitespace('\n'	)	)
											self.assertTrue(_is_whitespace('\u00A0'	)	)

											self.assertFalse(_is_whitespace('A'	)	)
											self.assertFalse(_is_whitespace('-'	)	)





							def 		__magic_name__			(    self	)->	List[Any]:
											self.assertTrue(_is_control('\u0005'	)	)

											self.assertFalse(_is_control('A'	)	)
											self.assertFalse(_is_control(' '	)	)
											self.assertFalse(_is_control('\t'	)	)
											self.assertFalse(_is_control('\r'	)	)





							def 		__magic_name__			(    self	)->	Dict:
											self.assertTrue(_is_punctuation('-'	)	)
											self.assertTrue(_is_punctuation('$'	)	)
											self.assertTrue(_is_punctuation('`'	)	)
											self.assertTrue(_is_punctuation('.'	)	)

											self.assertFalse(_is_punctuation('A'	)	)
											self.assertFalse(_is_punctuation(' '	)	)





							def 		__magic_name__			(    self	)->	str:
											_SCREAMING_SNAKE_CASE      = self.get_tokenizer()
											_SCREAMING_SNAKE_CASE      = self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(snake_case_	) for t in ['Test', '\xad', 'test']]				, [['[UNK]'], [], ['[UNK]']]	)

											self.assertListEqual(
											    [rust_tokenizer.tokenize(snake_case_	) for t in ['Test', '\xad', 'test']]				, [['[UNK]'], [], ['[UNK]']]	)





							@slow
							def 		__magic_name__			(    self	)->	Union[str, Any]:
											_SCREAMING_SNAKE_CASE      = self.tokenizer_class.from_pretrained('google/electra-base-discriminator'	)

											_SCREAMING_SNAKE_CASE      = tokenizer.encode('sequence builders'				, add_special_tokens=snake_case_	)
											_SCREAMING_SNAKE_CASE      = tokenizer.encode('multi-sequence build'				, add_special_tokens=snake_case_	)

											_SCREAMING_SNAKE_CASE      = tokenizer.build_inputs_with_special_tokens(snake_case_	)
											_SCREAMING_SNAKE_CASE      = tokenizer.build_inputs_with_special_tokens(snake_case_				, snake_case_	)

											assert encoded_sentence == [101] + text + [102]
											assert encoded_pair == [101] + text + [102] + text_a + [102]





							def 		__magic_name__			(    self	)->	Union[str, Any]:
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
																			_SCREAMING_SNAKE_CASE      = self.rust_tokenizer_class.from_pretrained(snake_case_				, **snake_case_	)

																			_SCREAMING_SNAKE_CASE      = F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																			_SCREAMING_SNAKE_CASE      = tokenizer_r.encode_plus(
																			    snake_case_				, return_attention_mask=snake_case_				, return_token_type_ids=snake_case_				, return_offsets_mapping=snake_case_				, add_special_tokens=snake_case_				, )

																			_SCREAMING_SNAKE_CASE      = tokenizer_r.do_lower_case if hasattr(snake_case_				, 'do_lower_case'	) else False
																			_SCREAMING_SNAKE_CASE      = (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), '''A'''),
																			        ((1, 2), ''','''),
																			        ((3, 5), '''na'''),
																			        ((5, 6), '''##ï'''),
																			        ((6, 8), '''##ve'''),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), '''Allen'''),
																			        ((21, 23), '''##NL'''),
																			        ((23, 24), '''##P'''),
																			        ((25, 33), '''sentence'''),
																			        ((33, 34), '''.'''),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), '''a'''),
																			        ((1, 2), ''','''),
																			        ((3, 8), '''naive'''),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), '''allen'''),
																			        ((21, 23), '''##nl'''),
																			        ((23, 24), '''##p'''),
																			        ((25, 33), '''sentence'''),
																			        ((33, 34), '''.'''),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]				, tokenizer_r.convert_ids_to_tokens(tokens['input_ids']	)	)
																			self.assertEqual([e[0] for e in expected_results]				, tokens['offset_mapping']	)





							def 		__magic_name__			(    self	)->	Optional[Any]:
											_SCREAMING_SNAKE_CASE      = ['''的''', '''人''', '''有''']
											_SCREAMING_SNAKE_CASE      = ''''''.join(snake_case_	)
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
																			_SCREAMING_SNAKE_CASE      = True
																			_SCREAMING_SNAKE_CASE      = self.tokenizer_class.from_pretrained(snake_case_				, **snake_case_	)
																			_SCREAMING_SNAKE_CASE      = self.rust_tokenizer_class.from_pretrained(snake_case_				, **snake_case_	)

																			_SCREAMING_SNAKE_CASE      = tokenizer_p.encode(snake_case_				, add_special_tokens=snake_case_	)
																			_SCREAMING_SNAKE_CASE      = tokenizer_r.encode(snake_case_				, add_special_tokens=snake_case_	)

																			_SCREAMING_SNAKE_CASE      = tokenizer_r.convert_ids_to_tokens(snake_case_	)
																			_SCREAMING_SNAKE_CASE      = tokenizer_p.convert_ids_to_tokens(snake_case_	)

																			# it is expected that each Chinese character is not preceded by "##"
																			self.assertListEqual(snake_case_				, snake_case_	)
																			self.assertListEqual(snake_case_				, snake_case_	)

																			_SCREAMING_SNAKE_CASE      = False
																			_SCREAMING_SNAKE_CASE      = self.rust_tokenizer_class.from_pretrained(snake_case_				, **snake_case_	)
																			_SCREAMING_SNAKE_CASE      = self.tokenizer_class.from_pretrained(snake_case_				, **snake_case_	)

																			_SCREAMING_SNAKE_CASE      = tokenizer_r.encode(snake_case_				, add_special_tokens=snake_case_	)
																			_SCREAMING_SNAKE_CASE      = tokenizer_p.encode(snake_case_				, add_special_tokens=snake_case_	)

																			_SCREAMING_SNAKE_CASE      = tokenizer_r.convert_ids_to_tokens(snake_case_	)
																			_SCREAMING_SNAKE_CASE      = tokenizer_p.convert_ids_to_tokens(snake_case_	)

																			# it is expected that only the first Chinese character is not preceded by "##".
																			_SCREAMING_SNAKE_CASE      = [
																			    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(snake_case_	)
																			]
																			self.assertListEqual(snake_case_				, snake_case_	)
																			self.assertListEqual(snake_case_				, snake_case_	)



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


snake_case			:      Optional[int] 							=   {'configuration_vit_hybrid': ['VIT_HYBRID_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ViTHybridConfig']}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							snake_case			:      int 							=   [
							    'VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'ViTHybridForImageClassification',
							    'ViTHybridModel',
							    'ViTHybridPreTrainedModel',
							]

try:
							if not is_vision_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							snake_case			:      Any 							=   ['ViTHybridImageProcessor']


if TYPE_CHECKING:
							from .configuration_vit_hybrid import VIT_HYBRID_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTHybridConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_vit_hybrid import (
														    VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST,
														    ViTHybridForImageClassification,
														    ViTHybridModel,
														    ViTHybridPreTrainedModel,
														)

							try:
														if not is_vision_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .image_processing_vit_hybrid import ViTHybridImageProcessor


else:
							import sys

							snake_case			:      Optional[int] 							=   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
