'''simple docstring'''

import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 		SCREAMING_SNAKE_CASE_				(_UpperCAmelCase       ,							unittest.TestCase			):



			"""simple docstring"""






			__magic_name__   :							Optional[int]             =					ElectraTokenizer
			__magic_name__   :							Union[str, Any]             =					ElectraTokenizerFast
			__magic_name__   :							Union[str, Any]             =					True
			__magic_name__   :							str             =					True
			__magic_name__   :							Optional[int]             =					filter_non_english



			def      lowerCamelCase   (       self				):
									super().setUp()

									snake_case   :					Union[str, Any]							=						[
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "[PAD]",
									    "[MASK]",
									    "want",
									    "##want",
									    "##ed",
									    "wa",
									    "un",
									    "runn",
									    "##ing",
									    ",",
									    "low",
									    "lowest",
									]
									snake_case   :					int							=						os.path.join(self.tmpdirname							, VOCAB_FILES_NAMES["vocab_file"]				)
									with open(self.vocab_file							, "w"							, encoding="utf-8"				) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)



			def      lowerCamelCase   (       self							, snake_case_				):
									snake_case   :					Union[str, Any]							=						"UNwant\u00E9d,running"
									snake_case   :					int							=						"unwanted, running"
									return input_text, output_text



			def      lowerCamelCase   (       self				):
									snake_case   :					int							=						self.tokenizer_class(self.vocab_file				)

									snake_case   :					List[str]							=						tokenizer.tokenize("UNwant\u00E9d,running"				)
									self.assertListEqual(snake_case_							, ["un", "##want", "##ed", ",", "runn", "##ing"]				)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_				)							, [9, 6, 7, 12, 10, 11]				)



			def      lowerCamelCase   (       self				):
									if not self.test_rust_tokenizer:
															return

									snake_case   :					Optional[int]							=						self.get_tokenizer()
									snake_case   :					Dict							=						self.get_rust_tokenizer()

									snake_case   :					Dict							=						"UNwant\u00E9d,running"

									snake_case   :					Union[str, Any]							=						tokenizer.tokenize(snake_case_				)
									snake_case   :					List[str]							=						rust_tokenizer.tokenize(snake_case_				)
									self.assertListEqual(snake_case_							, snake_case_				)

									snake_case   :					Optional[int]							=						tokenizer.encode(snake_case_							, add_special_tokens=snake_case_				)
									snake_case   :					Optional[Any]							=						rust_tokenizer.encode(snake_case_							, add_special_tokens=snake_case_				)
									self.assertListEqual(snake_case_							, snake_case_				)

									snake_case   :					Dict							=						self.get_rust_tokenizer()
									snake_case   :					List[Any]							=						tokenizer.encode(snake_case_				)
									snake_case   :					int							=						rust_tokenizer.encode(snake_case_				)
									self.assertListEqual(snake_case_							, snake_case_				)

									# With lower casing
									snake_case   :					List[Any]							=						self.get_tokenizer(do_lower_case=snake_case_				)
									snake_case   :					Union[str, Any]							=						self.get_rust_tokenizer(do_lower_case=snake_case_				)

									snake_case   :					int							=						"UNwant\u00E9d,running"

									snake_case   :					Dict							=						tokenizer.tokenize(snake_case_				)
									snake_case   :					Dict							=						rust_tokenizer.tokenize(snake_case_				)
									self.assertListEqual(snake_case_							, snake_case_				)

									snake_case   :					int							=						tokenizer.encode(snake_case_							, add_special_tokens=snake_case_				)
									snake_case   :					Any							=						rust_tokenizer.encode(snake_case_							, add_special_tokens=snake_case_				)
									self.assertListEqual(snake_case_							, snake_case_				)

									snake_case   :					List[str]							=						self.get_rust_tokenizer()
									snake_case   :					str							=						tokenizer.encode(snake_case_				)
									snake_case   :					Union[str, Any]							=						rust_tokenizer.encode(snake_case_				)
									self.assertListEqual(snake_case_							, snake_case_				)



			def      lowerCamelCase   (       self				):
									snake_case   :					Any							=						BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)							, ["ah", "\u535A", "\u63A8", "zz"]				)



			def      lowerCamelCase   (       self				):
									snake_case   :					Union[str, Any]							=						BasicTokenizer(do_lower_case=snake_case_				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)							, ["hello", "!", "how", "are", "you", "?"]				)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)							, ["hello"]				)



			def      lowerCamelCase   (       self				):
									snake_case   :					List[str]							=						BasicTokenizer(do_lower_case=snake_case_							, strip_accents=snake_case_				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)							, ["hällo", "!", "how", "are", "you", "?"]				)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)							, ["h\u00E9llo"]				)



			def      lowerCamelCase   (       self				):
									snake_case   :					List[Any]							=						BasicTokenizer(do_lower_case=snake_case_							, strip_accents=snake_case_				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)							, ["hallo", "!", "how", "are", "you", "?"]				)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)							, ["hello"]				)



			def      lowerCamelCase   (       self				):
									snake_case   :					Any							=						BasicTokenizer(do_lower_case=snake_case_				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)							, ["hallo", "!", "how", "are", "you", "?"]				)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)							, ["hello"]				)



			def      lowerCamelCase   (       self				):
									snake_case   :					Any							=						BasicTokenizer(do_lower_case=snake_case_				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)							, ["HeLLo", "!", "how", "Are", "yoU", "?"]				)



			def      lowerCamelCase   (       self				):
									snake_case   :					str							=						BasicTokenizer(do_lower_case=snake_case_							, strip_accents=snake_case_				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)							, ["HäLLo", "!", "how", "Are", "yoU", "?"]				)



			def      lowerCamelCase   (       self				):
									snake_case   :					int							=						BasicTokenizer(do_lower_case=snake_case_							, strip_accents=snake_case_				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)							, ["HaLLo", "!", "how", "Are", "yoU", "?"]				)



			def      lowerCamelCase   (       self				):
									snake_case   :					Union[str, Any]							=						BasicTokenizer(do_lower_case=snake_case_							, never_split=["[UNK]"]				)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)							, ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)



			def      lowerCamelCase   (       self				):
									snake_case   :					int							=						["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

									snake_case   :					Optional[int]							=						{}
									for i, token in enumerate(snake_case_				):
															snake_case   :					Dict							=						i
									snake_case   :					int							=						WordpieceTokenizer(vocab=snake_case_							, unk_token="[UNK]"				)

									self.assertListEqual(tokenizer.tokenize(""				)							, []				)

									self.assertListEqual(tokenizer.tokenize("unwanted running"				)							, ["un", "##want", "##ed", "runn", "##ing"]				)

									self.assertListEqual(tokenizer.tokenize("unwantedX running"				)							, ["[UNK]", "runn", "##ing"]				)



			def      lowerCamelCase   (       self				):
									self.assertTrue(_is_whitespace(" "				)				)
									self.assertTrue(_is_whitespace("\t"				)				)
									self.assertTrue(_is_whitespace("\r"				)				)
									self.assertTrue(_is_whitespace("\n"				)				)
									self.assertTrue(_is_whitespace("\u00A0"				)				)

									self.assertFalse(_is_whitespace("A"				)				)
									self.assertFalse(_is_whitespace("-"				)				)



			def      lowerCamelCase   (       self				):
									self.assertTrue(_is_control("\u0005"				)				)

									self.assertFalse(_is_control("A"				)				)
									self.assertFalse(_is_control(" "				)				)
									self.assertFalse(_is_control("\t"				)				)
									self.assertFalse(_is_control("\r"				)				)



			def      lowerCamelCase   (       self				):
									self.assertTrue(_is_punctuation("-"				)				)
									self.assertTrue(_is_punctuation("$"				)				)
									self.assertTrue(_is_punctuation("`"				)				)
									self.assertTrue(_is_punctuation("."				)				)

									self.assertFalse(_is_punctuation("A"				)				)
									self.assertFalse(_is_punctuation(" "				)				)



			def      lowerCamelCase   (       self				):
									snake_case   :					Union[str, Any]							=						self.get_tokenizer()
									snake_case   :					Tuple							=						self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(snake_case_				) for t in ["Test", "\xad", "test"]]							, [["[UNK]"], [], ["[UNK]"]]				)

									self.assertListEqual(
									    [rust_tokenizer.tokenize(snake_case_				) for t in ["Test", "\xad", "test"]]							, [["[UNK]"], [], ["[UNK]"]]				)



			@slow
			def      lowerCamelCase   (       self				):
									snake_case   :					str							=						self.tokenizer_class.from_pretrained("google/electra-base-discriminator"				)

									snake_case   :					List[Any]							=						tokenizer.encode("sequence builders"							, add_special_tokens=snake_case_				)
									snake_case   :					Dict							=						tokenizer.encode("multi-sequence build"							, add_special_tokens=snake_case_				)

									snake_case   :					Dict							=						tokenizer.build_inputs_with_special_tokens(snake_case_				)
									snake_case   :					Dict							=						tokenizer.build_inputs_with_special_tokens(snake_case_							, snake_case_				)

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]



			def      lowerCamelCase   (       self				):
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																					snake_case   :					Optional[int]							=						self.rust_tokenizer_class.from_pretrained(snake_case_							, **snake_case_				)

																					snake_case   :					Tuple							=						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																					snake_case   :					Any							=						tokenizer_r.encode_plus(
																					    snake_case_							, return_attention_mask=snake_case_							, return_token_type_ids=snake_case_							, return_offsets_mapping=snake_case_							, add_special_tokens=snake_case_							, )

																					snake_case   :					Tuple							=						tokenizer_r.do_lower_case if hasattr(snake_case_							, "do_lower_case"				) else False
																					snake_case   :					Union[str, Any]							=						(
																					    [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), "A"),
																					        ((1, 2), ","),
																					        ((3, 5), "na"),
																					        ((5, 6), "##ï"),
																					        ((6, 8), "##ve"),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), "Allen"),
																					        ((21, 23), "##NL"),
																					        ((23, 24), "##P"),
																					        ((25, 33), "sentence"),
																					        ((33, 34), "."),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					    if not do_lower_case
																					    else [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), "a"),
																					        ((1, 2), ","),
																					        ((3, 8), "naive"),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), "allen"),
																					        ((21, 23), "##nl"),
																					        ((23, 24), "##p"),
																					        ((25, 33), "sentence"),
																					        ((33, 34), "."),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					)

																					self.assertEqual(
																					    [e[1] for e in expected_results]							, tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
																					self.assertEqual([e[0] for e in expected_results]							, tokens["offset_mapping"]				)






			def      lowerCamelCase   (       self				):
									snake_case   :					Tuple							=						["的", "人", "有"]
									snake_case   :					Optional[Any]							=						"".join(snake_case_				)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
																					snake_case   :					Optional[int]							=						True
																					snake_case   :					Tuple							=						self.tokenizer_class.from_pretrained(snake_case_							, **snake_case_				)
																					snake_case   :					Union[str, Any]							=						self.rust_tokenizer_class.from_pretrained(snake_case_							, **snake_case_				)

																					snake_case   :					Optional[int]							=						tokenizer_p.encode(snake_case_							, add_special_tokens=snake_case_				)
																					snake_case   :					Any							=						tokenizer_r.encode(snake_case_							, add_special_tokens=snake_case_				)

																					snake_case   :					List[str]							=						tokenizer_r.convert_ids_to_tokens(snake_case_				)
																					snake_case   :					Optional[int]							=						tokenizer_p.convert_ids_to_tokens(snake_case_				)

																					# it is expected that each Chinese character is not preceded by "##"
																					self.assertListEqual(snake_case_							, snake_case_				)
																					self.assertListEqual(snake_case_							, snake_case_				)

																					snake_case   :					List[Any]							=						False
																					snake_case   :					Optional[int]							=						self.rust_tokenizer_class.from_pretrained(snake_case_							, **snake_case_				)
																					snake_case   :					List[str]							=						self.tokenizer_class.from_pretrained(snake_case_							, **snake_case_				)

																					snake_case   :					str							=						tokenizer_r.encode(snake_case_							, add_special_tokens=snake_case_				)
																					snake_case   :					str							=						tokenizer_p.encode(snake_case_							, add_special_tokens=snake_case_				)

																					snake_case   :					Optional[Any]							=						tokenizer_r.convert_ids_to_tokens(snake_case_				)
																					snake_case   :					Optional[Any]							=						tokenizer_p.convert_ids_to_tokens(snake_case_				)

																					# it is expected that only the first Chinese character is not preceded by "##".
																					snake_case   :					str							=						[
																					    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case_				)
																					]
																					self.assertListEqual(snake_case_							, snake_case_				)
																					self.assertListEqual(snake_case_							, snake_case_				)






'''simple docstring'''

from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A_        =   logging.get_logger(__name__)

A_        =   {
    "apple/mobilevit-small": "https://huggingface.co/apple/mobilevit-small/resolve/main/config.json",
    "apple/mobilevit-x-small": "https://huggingface.co/apple/mobilevit-x-small/resolve/main/config.json",
    "apple/mobilevit-xx-small": "https://huggingface.co/apple/mobilevit-xx-small/resolve/main/config.json",
    "apple/deeplabv3-mobilevit-small": (
        "https://huggingface.co/apple/deeplabv3-mobilevit-small/resolve/main/config.json"
    ),
    "apple/deeplabv3-mobilevit-x-small": (
        "https://huggingface.co/apple/deeplabv3-mobilevit-x-small/resolve/main/config.json"
    ),
    "apple/deeplabv3-mobilevit-xx-small": (
        "https://huggingface.co/apple/deeplabv3-mobilevit-xx-small/resolve/main/config.json"
    ),
    # See all MobileViT models at https://huggingface.co/models?filter=mobilevit
}




class 		SCREAMING_SNAKE_CASE_				(_UpperCAmelCase			):



			"""simple docstring"""






			__magic_name__   :							Optional[Any]             =					"""mobilevit"""



			def __init__(       self							, snake_case_=3							, snake_case_=256							, snake_case_=2							, snake_case_=[144, 192, 240]							, snake_case_=[16, 32, 64, 96, 128, 160, 640]							, snake_case_=4							, snake_case_=2.0							, snake_case_=4.0							, snake_case_="silu"							, snake_case_=3							, snake_case_=32							, snake_case_=0.1							, snake_case_=0.0							, snake_case_=0.1							, snake_case_=0.02							, snake_case_=1e-5							, snake_case_=True							, snake_case_=256							, snake_case_=[6, 12, 18]							, snake_case_=0.1							, snake_case_=255							, **snake_case_							, ):
									super().__init__(**snake_case_				)

									snake_case   :					Any							=						num_channels
									snake_case   :					Optional[Any]							=						image_size
									snake_case   :					Union[str, Any]							=						patch_size
									snake_case   :					Union[str, Any]							=						hidden_sizes
									snake_case   :					List[Any]							=						neck_hidden_sizes
									snake_case   :					Optional[Any]							=						num_attention_heads
									snake_case   :					Any							=						mlp_ratio
									snake_case   :					List[str]							=						expand_ratio
									snake_case   :					Tuple							=						hidden_act
									snake_case   :					Tuple							=						conv_kernel_size
									snake_case   :					int							=						output_stride
									snake_case   :					Dict							=						hidden_dropout_prob
									snake_case   :					List[str]							=						attention_probs_dropout_prob
									snake_case   :					int							=						classifier_dropout_prob
									snake_case   :					str							=						initializer_range
									snake_case   :					Optional[int]							=						layer_norm_eps
									snake_case   :					Dict							=						qkv_bias

									# decode head attributes for semantic segmentation
									snake_case   :					Optional[int]							=						aspp_out_channels
									snake_case   :					Dict							=						atrous_rates
									snake_case   :					Any							=						aspp_dropout_prob
									snake_case   :					Any							=						semantic_loss_ignore_index







class 		SCREAMING_SNAKE_CASE_				(_UpperCAmelCase			):



			"""simple docstring"""






			__magic_name__   :							Union[str, Any]             =					version.parse("""1.11"""			)



			@property
			def      lowerCamelCase   (       self				):
									return OrderedDict([("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"})]				)



			@property
			def      lowerCamelCase   (       self				):
									if self.task == "image-classification":
															return OrderedDict([("logits", {0: "batch"})]				)
									else:
															return OrderedDict([("last_hidden_state", {0: "batch"}), ("pooler_output", {0: "batch"})]				)






			@property
			def      lowerCamelCase   (       self				):
									return 1e-4





