



import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 	a__       (       _snake_case  ,			unittest.TestCase       ):


					"""simple docstring"""



					A__  :				List[str]					 =				ElectraTokenizer
					A__  :				Tuple					 =				ElectraTokenizerFast
					A__  :				Optional[Any]					 =				True
					A__  :				Any					 =				True
					A__  :				str					 =				filter_non_english

					def 				__UpperCAmelCase		(		self     :int    ):
										super().setUp()

										lowercase						   =       [
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										lowercase						   =       os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES['vocab_file']    )
										with open(self.vocab_file      ,  'w'      ,  encoding='utf-8'    ) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )

					def 				__UpperCAmelCase		(		self     :Optional[int]      ,  lowercase__     :List[Any]    ):
										lowercase						   =       'UNwant\u00E9d,running'
										lowercase						   =       'unwanted, running'
										return input_text, output_text

					def 				__UpperCAmelCase		(		self     :Tuple    ):
										lowercase						   =       self.tokenizer_class(self.vocab_file    )

										lowercase						   =       tokenizer.tokenize('UNwant\u00E9d,running'    )
										self.assertListEqual(lowercase__      ,  ['un', '##want', '##ed', ',', 'runn', '##ing']    )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__    )      ,  [9, 6, 7, 12, 10, 11]    )

					def 				__UpperCAmelCase		(		self     :List[str]    ):
										if not self.test_rust_tokenizer:
															return

										lowercase						   =       self.get_tokenizer()
										lowercase						   =       self.get_rust_tokenizer()

										lowercase						   =       'UNwant\u00E9d,running'

										lowercase						   =       tokenizer.tokenize(lowercase__    )
										lowercase						   =       rust_tokenizer.tokenize(lowercase__    )
										self.assertListEqual(lowercase__      ,  lowercase__    )

										lowercase						   =       tokenizer.encode(lowercase__      ,  add_special_tokens=lowercase__    )
										lowercase						   =       rust_tokenizer.encode(lowercase__      ,  add_special_tokens=lowercase__    )
										self.assertListEqual(lowercase__      ,  lowercase__    )

										lowercase						   =       self.get_rust_tokenizer()
										lowercase						   =       tokenizer.encode(lowercase__    )
										lowercase						   =       rust_tokenizer.encode(lowercase__    )
										self.assertListEqual(lowercase__      ,  lowercase__    )

										# With lower casing
										lowercase						   =       self.get_tokenizer(do_lower_case=lowercase__    )
										lowercase						   =       self.get_rust_tokenizer(do_lower_case=lowercase__    )

										lowercase						   =       'UNwant\u00E9d,running'

										lowercase						   =       tokenizer.tokenize(lowercase__    )
										lowercase						   =       rust_tokenizer.tokenize(lowercase__    )
										self.assertListEqual(lowercase__      ,  lowercase__    )

										lowercase						   =       tokenizer.encode(lowercase__      ,  add_special_tokens=lowercase__    )
										lowercase						   =       rust_tokenizer.encode(lowercase__      ,  add_special_tokens=lowercase__    )
										self.assertListEqual(lowercase__      ,  lowercase__    )

										lowercase						   =       self.get_rust_tokenizer()
										lowercase						   =       tokenizer.encode(lowercase__    )
										lowercase						   =       rust_tokenizer.encode(lowercase__    )
										self.assertListEqual(lowercase__      ,  lowercase__    )

					def 				__UpperCAmelCase		(		self     :Optional[Any]    ):
										lowercase						   =       BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'    )      ,  ['ah', '\u535A', '\u63A8', 'zz']    )

					def 				__UpperCAmelCase		(		self     :Optional[Any]    ):
										lowercase						   =       BasicTokenizer(do_lower_case=lowercase__    )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )      ,  ['hello', '!', 'how', 'are', 'you', '?']    )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )      ,  ['hello']    )

					def 				__UpperCAmelCase		(		self     :Optional[int]    ):
										lowercase						   =       BasicTokenizer(do_lower_case=lowercase__      ,  strip_accents=lowercase__    )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )      ,  ['hällo', '!', 'how', 'are', 'you', '?']    )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )      ,  ['h\u00E9llo']    )

					def 				__UpperCAmelCase		(		self     :Tuple    ):
										lowercase						   =       BasicTokenizer(do_lower_case=lowercase__      ,  strip_accents=lowercase__    )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )      ,  ['hallo', '!', 'how', 'are', 'you', '?']    )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )      ,  ['hello']    )

					def 				__UpperCAmelCase		(		self     :Optional[Any]    ):
										lowercase						   =       BasicTokenizer(do_lower_case=lowercase__    )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )      ,  ['hallo', '!', 'how', 'are', 'you', '?']    )
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )      ,  ['hello']    )

					def 				__UpperCAmelCase		(		self     :Optional[Any]    ):
										lowercase						   =       BasicTokenizer(do_lower_case=lowercase__    )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )      ,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?']    )

					def 				__UpperCAmelCase		(		self     :List[str]    ):
										lowercase						   =       BasicTokenizer(do_lower_case=lowercase__      ,  strip_accents=lowercase__    )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )      ,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?']    )

					def 				__UpperCAmelCase		(		self     :Optional[int]    ):
										lowercase						   =       BasicTokenizer(do_lower_case=lowercase__      ,  strip_accents=lowercase__    )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )      ,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?']    )

					def 				__UpperCAmelCase		(		self     :Optional[Any]    ):
										lowercase						   =       BasicTokenizer(do_lower_case=lowercase__      ,  never_split=['[UNK]']    )

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'    )      ,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']    )

					def 				__UpperCAmelCase		(		self     :Optional[int]    ):
										lowercase						   =       ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										lowercase						   =       {}
										for i, token in enumerate(lowercase__    ):
															lowercase						   =       i
										lowercase						   =       WordpieceTokenizer(vocab=lowercase__      ,  unk_token='[UNK]'    )

										self.assertListEqual(tokenizer.tokenize(''    )      ,  []    )

										self.assertListEqual(tokenizer.tokenize('unwanted running'    )      ,  ['un', '##want', '##ed', 'runn', '##ing']    )

										self.assertListEqual(tokenizer.tokenize('unwantedX running'    )      ,  ['[UNK]', 'runn', '##ing']    )

					def 				__UpperCAmelCase		(		self     :Any    ):
										self.assertTrue(_is_whitespace(' '    )    )
										self.assertTrue(_is_whitespace('\t'    )    )
										self.assertTrue(_is_whitespace('\r'    )    )
										self.assertTrue(_is_whitespace('\n'    )    )
										self.assertTrue(_is_whitespace('\u00A0'    )    )

										self.assertFalse(_is_whitespace('A'    )    )
										self.assertFalse(_is_whitespace('-'    )    )

					def 				__UpperCAmelCase		(		self     :Dict    ):
										self.assertTrue(_is_control('\u0005'    )    )

										self.assertFalse(_is_control('A'    )    )
										self.assertFalse(_is_control(' '    )    )
										self.assertFalse(_is_control('\t'    )    )
										self.assertFalse(_is_control('\r'    )    )

					def 				__UpperCAmelCase		(		self     :str    ):
										self.assertTrue(_is_punctuation('-'    )    )
										self.assertTrue(_is_punctuation('$'    )    )
										self.assertTrue(_is_punctuation('`'    )    )
										self.assertTrue(_is_punctuation('.'    )    )

										self.assertFalse(_is_punctuation('A'    )    )
										self.assertFalse(_is_punctuation(' '    )    )

					def 				__UpperCAmelCase		(		self     :Dict    ):
										lowercase						   =       self.get_tokenizer()
										lowercase						   =       self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowercase__    ) for t in ['Test', '\xad', 'test']]      ,  [['[UNK]'], [], ['[UNK]']]    )

										self.assertListEqual(
										    [rust_tokenizer.tokenize(lowercase__    ) for t in ['Test', '\xad', 'test']]      ,  [['[UNK]'], [], ['[UNK]']]    )

					@slow
					def 				__UpperCAmelCase		(		self     :List[str]    ):
										lowercase						   =       self.tokenizer_class.from_pretrained('google/electra-base-discriminator'    )

										lowercase						   =       tokenizer.encode('sequence builders'      ,  add_special_tokens=lowercase__    )
										lowercase						   =       tokenizer.encode('multi-sequence build'      ,  add_special_tokens=lowercase__    )

										lowercase						   =       tokenizer.build_inputs_with_special_tokens(lowercase__    )
										lowercase						   =       tokenizer.build_inputs_with_special_tokens(lowercase__      ,  lowercase__    )

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]

					def 				__UpperCAmelCase		(		self     :Optional[Any]    ):
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
																				lowercase						   =       self.rust_tokenizer_class.from_pretrained(lowercase__      ,  **lowercase__    )

																				lowercase						   =       F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																				lowercase						   =       tokenizer_r.encode_plus(
																				    lowercase__      ,  return_attention_mask=lowercase__      ,  return_token_type_ids=lowercase__      ,  return_offsets_mapping=lowercase__      ,  add_special_tokens=lowercase__      ,  )

																				lowercase						   =       tokenizer_r.do_lower_case if hasattr(lowercase__      ,  'do_lower_case'    ) else False
																				lowercase						   =       (
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'A'),
																				        ((1, 2), ','),
																				        ((3, 5), 'na'),
																				        ((5, 6), '##ï'),
																				        ((6, 8), '##ve'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'Allen'),
																				        ((21, 23), '##NL'),
																				        ((23, 24), '##P'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'a'),
																				        ((1, 2), ','),
																				        ((3, 8), 'naive'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'allen'),
																				        ((21, 23), '##nl'),
																				        ((23, 24), '##p'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]      ,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids']    )    )
																				self.assertEqual([e[0] for e in expected_results]      ,  tokens['offset_mapping']    )



					def 				__UpperCAmelCase		(		self     :List[str]    ):
										lowercase						   =       ['的', '人', '有']
										lowercase						   =       ''.join(lowercase__    )
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""    ):
																				lowercase						   =       True
																				lowercase						   =       self.tokenizer_class.from_pretrained(lowercase__      ,  **lowercase__    )
																				lowercase						   =       self.rust_tokenizer_class.from_pretrained(lowercase__      ,  **lowercase__    )

																				lowercase						   =       tokenizer_p.encode(lowercase__      ,  add_special_tokens=lowercase__    )
																				lowercase						   =       tokenizer_r.encode(lowercase__      ,  add_special_tokens=lowercase__    )

																				lowercase						   =       tokenizer_r.convert_ids_to_tokens(lowercase__    )
																				lowercase						   =       tokenizer_p.convert_ids_to_tokens(lowercase__    )

																				# it is expected that each Chinese character is not preceded by "##"
																				self.assertListEqual(lowercase__      ,  lowercase__    )
																				self.assertListEqual(lowercase__      ,  lowercase__    )

																				lowercase						   =       False
																				lowercase						   =       self.rust_tokenizer_class.from_pretrained(lowercase__      ,  **lowercase__    )
																				lowercase						   =       self.tokenizer_class.from_pretrained(lowercase__      ,  **lowercase__    )

																				lowercase						   =       tokenizer_r.encode(lowercase__      ,  add_special_tokens=lowercase__    )
																				lowercase						   =       tokenizer_p.encode(lowercase__      ,  add_special_tokens=lowercase__    )

																				lowercase						   =       tokenizer_r.convert_ids_to_tokens(lowercase__    )
																				lowercase						   =       tokenizer_p.convert_ids_to_tokens(lowercase__    )

																				# it is expected that only the first Chinese character is not preceded by "##".
																				lowercase						   =       [
																				    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowercase__    )
																				]
																				self.assertListEqual(lowercase__      ,  lowercase__    )
																				self.assertListEqual(lowercase__      ,  lowercase__    )











import os

from packaging import version

from .. import __version__
from .accelerate_utils import apply_forward_hook
from .constants import (
    CONFIG_NAME,
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    DIFFUSERS_DYNAMIC_MODULE_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    ONNX_EXTERNAL_WEIGHTS_NAME,
    ONNX_WEIGHTS_NAME,
    SAFETENSORS_WEIGHTS_NAME,
    TEXT_ENCODER_ATTN_MODULE,
    WEIGHTS_NAME,
)
from .deprecation_utils import deprecate
from .doc_utils import replace_example_docstring
from .dynamic_modules_utils import get_class_from_dynamic_module
from .hub_utils import (
    HF_HUB_OFFLINE,
    _add_variant,
    _get_model_file,
    extract_commit_hash,
    http_user_agent,
)
from .import_utils import (
    BACKENDS_MAPPING,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    is_accelerate_available,
    is_accelerate_version,
    is_bsa_available,
    is_flax_available,
    is_ftfy_available,
    is_inflect_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_k_diffusion_version,
    is_librosa_available,
    is_note_seq_available,
    is_omegaconf_available,
    is_onnx_available,
    is_safetensors_available,
    is_scipy_available,
    is_tensorboard_available,
    is_tf_available,
    is_torch_available,
    is_torch_version,
    is_torchsde_available,
    is_transformers_available,
    is_transformers_version,
    is_unidecode_available,
    is_wandb_available,
    is_xformers_available,
    requires_backends,
)
from .logging import get_logger
from .outputs import BaseOutput
from .pil_utils import PIL_INTERPOLATION, numpy_to_pil, pt_to_pil
from .torch_utils import is_compiled_module, randn_tensor


if is_torch_available():
		from .testing_utils import (
		    floats_tensor,
		    load_hf_numpy,
		    load_image,
		    load_numpy,
		    load_pt,
		    nightly,
		    parse_flag_from_env,
		    print_tensor_test,
		    require_torch_a,
		    require_torch_gpu,
		    skip_mps,
		    slow,
		    torch_all_close,
		    torch_device,
		)
		from .torch_utils import maybe_allow_in_graph

from .testing_utils import export_to_gif, export_to_video


__magic_name__       =     get_logger(__name__)





def 		__snake_case						(    _UpperCAmelCase				):
					"""simple docstring"""






					if version.parse(_UpperCAmelCase				) < version.parse(_UpperCAmelCase				):
										if "dev" in min_version:
															lowercase						   =       (
															    'This example requires a source install from HuggingFace diffusers (see '
															    '`https://huggingface.co/docs/diffusers/installation#install-from-source`),'
															)
										else:
															lowercase						   =       f"""This example requires a minimum version of {min_version},"""
										error_message += f""" but the version found is {__version__}.\n"""
										raise ImportError(_UpperCAmelCase				)






