






"""simple docstring"""

import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation

lowercase__      =				logging.get_logger(__name__)

lowercase__      =				{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowercase__      =				{
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
}

lowercase__      =				{
    """gpt2""": 1024,
    """gpt2-medium""": 1024,
    """gpt2-large""": 1024,
    """gpt2-xl""": 1024,
    """distilgpt2""": 1024,
}

@lru_cache()
def 	__lowerCamelCase			(						)		->					Dict:

		"""simple docstring"""







		lowerCAmelCase_     :							List[str]             =    (
		    list(range(ord("!" )   , ord("~" ) + 1 ) ) + list(range(ord("¡" )   , ord("¬" ) + 1 ) ) + list(range(ord("®" )   , ord("ÿ" ) + 1 ) )
		)
		lowerCAmelCase_     :							str             =    bs[:]
		lowerCAmelCase_     :							List[Any]             =    0
		for b in range(2**8 ):
				if b not in bs:
						bs.append(__UpperCamelCase )
						cs.append(2**8 + n )
						n += 1
		lowerCAmelCase_     :							List[str]             =    [chr(__UpperCamelCase ) for n in cs]
		return dict(zip(__UpperCamelCase   , __UpperCamelCase ) )

def 	__lowerCamelCase			(						__UpperCamelCase )		->					Union[str, Any]:

		"""simple docstring"""







		lowerCAmelCase_     :							str             =    set()
		lowerCAmelCase_     :							Union[str, Any]             =    word[0]
		for char in word[1:]:
				pairs.add((prev_char, char) )
				lowerCAmelCase_     :							Union[str, Any]             =    char
		return pairs



class 	__lowerCamelCase       (				A__    ):





		'''simple docstring'''






		a_			:       str      =							VOCAB_FILES_NAMES
		a_			:       List[Any]      =							PRETRAINED_VOCAB_FILES_MAP
		a_			:       int      =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		a_			:       Union[str, Any]      =							["""input_ids""", """attention_mask"""]





		def __init__(     self							:			Dict		,    a_							:			Union[str, Any]		,    a_							:			Union[str, Any]		,    a_							:			List[str]="replace"		,    a_							:			Union[str, Any]="<|endoftext|>"		,    a_							:			Union[str, Any]="<|endoftext|>"		,    a_							:			Optional[int]="<|endoftext|>"		,    a_							:			Any=None		,    a_							:			Optional[int]=False		,    a_							:			Union[str, Any]=False		,    **a_							:			int		,    ):
				lowerCAmelCase_     :							Optional[int]             =    AddedToken(a_		,    lstrip=a_		,    rstrip=a_			) if isinstance(a_		,    a_			) else bos_token
				lowerCAmelCase_     :							int             =    AddedToken(a_		,    lstrip=a_		,    rstrip=a_			) if isinstance(a_		,    a_			) else eos_token
				lowerCAmelCase_     :							Optional[int]             =    AddedToken(a_		,    lstrip=a_		,    rstrip=a_			) if isinstance(a_		,    a_			) else unk_token
				lowerCAmelCase_     :							Any             =    AddedToken(a_		,    lstrip=a_		,    rstrip=a_			) if isinstance(a_		,    a_			) else pad_token
				super().__init__(
				    errors=a_		,    unk_token=a_		,    bos_token=a_		,    eos_token=a_		,    pad_token=a_		,    add_prefix_space=a_		,    add_bos_token=a_		,    **a_		,    )
				lowerCAmelCase_     :							Union[str, Any]             =    add_bos_token

				with open(a_		,    encoding="utf-8"			) as vocab_handle:
						lowerCAmelCase_     :							Tuple             =    json.load(a_			)
				lowerCAmelCase_     :							Optional[int]             =    {v: k for k, v in self.encoder.items()}
				lowerCAmelCase_     :							str             =    errors  # how to handle errors in decoding
				lowerCAmelCase_     :							Optional[Any]             =    bytes_to_unicode()
				lowerCAmelCase_     :							str             =    {v: k for k, v in self.byte_encoder.items()}
				with open(a_		,    encoding="utf-8"			) as merges_handle:
						lowerCAmelCase_     :							Tuple             =    merges_handle.read().split("\n"			)[1:-1]
				lowerCAmelCase_     :							str             =    [tuple(merge.split()			) for merge in bpe_merges]
				lowerCAmelCase_     :							List[str]             =    dict(zip(a_		,    range(len(a_			)			)			)			)
				lowerCAmelCase_     :							Union[str, Any]             =    {}
				lowerCAmelCase_     :							List[str]             =    add_prefix_space

				# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
				lowerCAmelCase_     :							str             =    re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"			)





		@property
		def      lowerCamelCase (     self							:			Any			):
				return len(self.encoder			)





		def      lowerCamelCase (     self							:			Tuple			):
				return dict(self.encoder		,    **self.added_tokens_encoder			)





		def      lowerCamelCase (     self							:			Dict		,    a_							:			Dict			):
				if token in self.cache:
						return self.cache[token]
				lowerCAmelCase_     :							int             =    tuple(a_			)
				lowerCAmelCase_     :							Any             =    get_pairs(a_			)

				if not pairs:
						return token

				while True:
						lowerCAmelCase_     :							List[Any]             =    min(a_		,    key=lambda a_			: self.bpe_ranks.get(a_		,    float("inf"			)			)			)
						if bigram not in self.bpe_ranks:
								break
						lowerCAmelCase_       ,			lowerCAmelCase_     :							int             =    bigram
						lowerCAmelCase_     :							List[str]             =    []
						lowerCAmelCase_     :							Union[str, Any]             =    0
						while i < len(a_			):
								try:
										lowerCAmelCase_     :							Optional[Any]             =    word.index(a_		,    a_			)
								except ValueError:
										new_word.extend(word[i:]			)
										break
								else:
										new_word.extend(word[i:j]			)
										lowerCAmelCase_     :							Any             =    j

								if word[i] == first and i < len(a_			) - 1 and word[i + 1] == second:
										new_word.append(first + second			)
										i += 2
								else:
										new_word.append(word[i]			)
										i += 1
						lowerCAmelCase_     :							Any             =    tuple(a_			)
						lowerCAmelCase_     :							Optional[int]             =    new_word
						if len(a_			) == 1:
								break
						else:
								lowerCAmelCase_     :							Any             =    get_pairs(a_			)
				lowerCAmelCase_     :							Optional[int]             =    " ".join(a_			)
				lowerCAmelCase_     :							int             =    word
				return word





		def      lowerCamelCase (     self							:			List[str]		,    a_							:			Union[str, Any]		,    a_							:			List[Any]=None			):
				if self.add_bos_token:
						lowerCAmelCase_     :							Union[str, Any]             =    [self.bos_token_id]
				else:
						lowerCAmelCase_     :							Dict             =    []

				lowerCAmelCase_     :							Union[str, Any]             =    bos_token_ids + token_ids_a

				if token_ids_a is None:
						return output

				return output + bos_token_ids + token_ids_a





		def      lowerCamelCase (     self							:			List[Any]		,    a_							:			List[int]		,    a_							:			Optional[List[int]] = None		,    a_							:			bool = False			):
				if already_has_special_tokens:
						return super().get_special_tokens_mask(
						    token_ids_a=a_		,    token_ids_a=a_		,    already_has_special_tokens=a_			)

				if not self.add_bos_token:
						return super().get_special_tokens_mask(
						    token_ids_a=a_		,    token_ids_a=a_		,    already_has_special_tokens=a_			)

				if token_ids_a is None:
						return [1] + ([0] * len(a_			))
				return [1] + ([0] * len(a_			)) + [1] + ([0] * len(a_			))





		def      lowerCamelCase (     self							:			Optional[Any]		,    a_							:			Optional[Any]			):
				lowerCAmelCase_     :							Any             =    []
				for token in re.findall(self.pat		,    a_			):
						lowerCAmelCase_     :							Optional[Any]             =    "".join(
						    self.byte_encoder[b] for b in token.encode("utf-8"			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
						bpe_tokens.extend(bpe_token for bpe_token in self.bpe(a_			).split(" "			)			)
				return bpe_tokens





		def      lowerCamelCase (     self							:			Optional[Any]		,    a_							:			Any			):
				return self.encoder.get(a_		,    self.encoder.get(self.unk_token			)			)





		def      lowerCamelCase (     self							:			Optional[int]		,    a_							:			int			):
				return self.decoder.get(a_			)





		def      lowerCamelCase (     self							:			Any		,    a_							:			List[str]			):
				lowerCAmelCase_     :							Optional[Any]             =    "".join(a_			)
				lowerCAmelCase_     :							Tuple             =    bytearray([self.byte_decoder[c] for c in text]			).decode("utf-8"		,    errors=self.errors			)
				return text





		def      lowerCamelCase (     self							:			int		,    a_							:			str		,    a_							:			Optional[str] = None			):
				if not os.path.isdir(a_			):
						logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''			)
						return
				lowerCAmelCase_     :							Optional[Any]             =    os.path.join(
				    a_		,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
				lowerCAmelCase_     :							Optional[int]             =    os.path.join(
				    a_		,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

				with open(a_		,    "w"		,    encoding="utf-8"			) as f:
						f.write(json.dumps(self.encoder		,    indent=2		,    sort_keys=a_		,    ensure_ascii=a_			) + "\n"			)

				lowerCAmelCase_     :							Optional[int]             =    0
				with open(a_		,    "w"		,    encoding="utf-8"			) as writer:
						writer.write("#version: 0.2\n"			)
						for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,    key=lambda a_			: kv[1]			):
								if index != token_index:
										logger.warning(
										    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
										    " Please check that the tokenizer is not corrupted!"			)
										lowerCAmelCase_     :							Optional[Any]             =    token_index
								writer.write(" ".join(a_			) + "\n"			)
								index += 1

				return vocab_file, merge_file





		def      lowerCamelCase (     self							:			Dict		,    a_							:			Dict		,    a_							:			int=False		,    **a_							:			str			):
				lowerCAmelCase_     :							int             =    kwargs.pop("add_prefix_space"		,    self.add_prefix_space			)
				if is_split_into_words or add_prefix_space:
						lowerCAmelCase_     :							str             =    " " + text
				return (text, kwargs)





		def      lowerCamelCase (     self							:			Dict		,    a_							:			"Conversation"			):
				lowerCAmelCase_     :							Union[str, Any]             =    []
				for is_user, text in conversation.iter_texts():
						input_ids.extend(self.encode(a_		,    add_special_tokens=a_			) + [self.eos_token_id]			)
				if len(a_			) > self.model_max_length:
						lowerCAmelCase_     :							Optional[int]             =    input_ids[-self.model_max_length :]
				return input_ids








"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
    is_vision_available,
)


lowercase__      =				{
    """configuration_mobilevitv2""": [
        """MOBILEVITV2_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """MobileViTV2Config""",
        """MobileViTV2OnnxConfig""",
    ],
}


try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase__      =				[
					    """MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """MobileViTV2ForImageClassification""",
					    """MobileViTV2ForSemanticSegmentation""",
					    """MobileViTV2Model""",
					    """MobileViTV2PreTrainedModel""",
					]

if TYPE_CHECKING:
					from .configuration_mobilevitva import (
					    MOBILEVITV2_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    MobileViTVaConfig,
					    MobileViTVaOnnxConfig,
					)

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_mobilevitva import (
										    MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST,
										    MobileViTVaForImageClassification,
										    MobileViTVaForSemanticSegmentation,
										    MobileViTVaModel,
										    MobileViTVaPreTrainedModel,
										)

else:
					import sys

					lowercase__      =				_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
