"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase_							:							Tuple				 =		logging.get_logger(__name__)


UpperCamelCase_							:							Optional[Any]				 =		{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
UpperCamelCase_							:							List[str]				 =		{
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
}

UpperCamelCase_							:							Optional[Any]				 =		{
    """RUCAIBox/mvp""": 1024,
}


@lru_cache()
def 	a__   (		)->						Dict:
						UpperCAmelCase__       						=		(
						    list(range(ord("""!"""      )		,      ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      )		,      ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      )		,      ord("""ÿ"""      ) + 1      )      )
						)
						UpperCAmelCase__       						=		bs[:]
						UpperCAmelCase__       						=		0
						for b in range(2**8      ):
												if b not in bs:
																		bs.append(UpperCAmelCase__      )
																		cs.append(2**8 + n      )
																		n += 1
						UpperCAmelCase__       						=		[chr(UpperCAmelCase__      ) for n in cs]
						return dict(zip(UpperCAmelCase__		,      UpperCAmelCase__      )      )


def 	a__   (		UpperCAmelCase__							:      List[Any]      )->						Dict:
						UpperCAmelCase__       						=		set()
						UpperCAmelCase__       						=		word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)      )
												UpperCAmelCase__       						=		char
						return pairs







class       _lowercase		( __lowerCamelCase	):
				lowerCamelCase__							=							VOCAB_FILES_NAMES
				lowerCamelCase__							=							PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase__							=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase__							=							['input_ids', 'attention_mask']

				def __init__(  self							:     Any							,     _lowercase							:     Optional[int]							,     _lowercase							:     Union[str, Any]							,     _lowercase							:     Union[str, Any]="replace"							,     _lowercase							:     str="<s>"							,     _lowercase							:     Optional[Any]="</s>"							,     _lowercase							:     int="</s>"							,     _lowercase							:     List[str]="<s>"							,     _lowercase							:     Union[str, Any]="<unk>"							,     _lowercase							:     int="<pad>"							,     _lowercase							:     Optional[int]="<mask>"							,     _lowercase							:     Union[str, Any]=False							,     **_lowercase							:     Optional[Any]							,     ):


										'''simple docstring'''
										UpperCAmelCase__       						=		AddedToken(_lowercase							,     lstrip=_lowercase							,     rstrip=_lowercase  ) if isinstance(_lowercase							,     _lowercase  ) else bos_token
										UpperCAmelCase__       						=		AddedToken(_lowercase							,     lstrip=_lowercase							,     rstrip=_lowercase  ) if isinstance(_lowercase							,     _lowercase  ) else eos_token
										UpperCAmelCase__       						=		AddedToken(_lowercase							,     lstrip=_lowercase							,     rstrip=_lowercase  ) if isinstance(_lowercase							,     _lowercase  ) else sep_token
										UpperCAmelCase__       						=		AddedToken(_lowercase							,     lstrip=_lowercase							,     rstrip=_lowercase  ) if isinstance(_lowercase							,     _lowercase  ) else cls_token
										UpperCAmelCase__       						=		AddedToken(_lowercase							,     lstrip=_lowercase							,     rstrip=_lowercase  ) if isinstance(_lowercase							,     _lowercase  ) else unk_token
										UpperCAmelCase__       						=		AddedToken(_lowercase							,     lstrip=_lowercase							,     rstrip=_lowercase  ) if isinstance(_lowercase							,     _lowercase  ) else pad_token

										# Mask token behave like a normal word, i.e. include the space before it
										UpperCAmelCase__       						=		AddedToken(_lowercase							,     lstrip=_lowercase							,     rstrip=_lowercase  ) if isinstance(_lowercase							,     _lowercase  ) else mask_token

										super().__init__(
										    errors=_lowercase							,     bos_token=_lowercase							,     eos_token=_lowercase							,     unk_token=_lowercase							,     sep_token=_lowercase							,     cls_token=_lowercase							,     pad_token=_lowercase							,     mask_token=_lowercase							,     add_prefix_space=_lowercase							,     **_lowercase							,     )
										with open(_lowercase							,     encoding="""utf-8"""  ) as vocab_handle:
																UpperCAmelCase__       						=		json.load(_lowercase  )
										UpperCAmelCase__       						=		{v: k for k, v in self.encoder.items()}
										UpperCAmelCase__       						=		errors  # how to handle errors in decoding
										UpperCAmelCase__       						=		bytes_to_unicode()
										UpperCAmelCase__       						=		{v: k for k, v in self.byte_encoder.items()}
										with open(_lowercase							,     encoding="""utf-8"""  ) as merges_handle:
																UpperCAmelCase__       						=		merges_handle.read().split("""\n"""  )[1:-1]
										UpperCAmelCase__       						=		[tuple(merge.split()  ) for merge in bpe_merges]
										UpperCAmelCase__       						=		dict(zip(_lowercase							,     range(len(_lowercase  )  )  )  )
										UpperCAmelCase__       						=		{}
										UpperCAmelCase__       						=		add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										UpperCAmelCase__       						=		re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""  )

				@property
				def 							A			(  self							:     Dict  ):


										'''simple docstring'''
										return len(self.encoder  )

				def 							A			(  self							:     Union[str, Any]  ):


										'''simple docstring'''
										return dict(self.encoder							,     **self.added_tokens_encoder  )

				def 							A			(  self							:     Union[str, Any]							,     _lowercase							:     str  ):


										'''simple docstring'''
										if token in self.cache:
																return self.cache[token]
										UpperCAmelCase__       						=		tuple(_lowercase  )
										UpperCAmelCase__       						=		get_pairs(_lowercase  )

										if not pairs:
																return token

										while True:
																UpperCAmelCase__       						=		min(_lowercase							,     key=lambda _lowercase  : self.bpe_ranks.get(_lowercase							,     float("""inf"""  )  )  )
																if bigram not in self.bpe_ranks:
																						break
																UpperCAmelCase__						,   UpperCAmelCase__       						=		bigram
																UpperCAmelCase__       						=		[]
																UpperCAmelCase__       						=		0
																while i < len(_lowercase  ):
																						try:
																												UpperCAmelCase__       						=		word.index(_lowercase							,     _lowercase  )
																						except ValueError:
																												new_word.extend(word[i:]  )
																												break
																						else:
																												new_word.extend(word[i:j]  )
																												UpperCAmelCase__       						=		j

																						if word[i] == first and i < len(_lowercase  ) - 1 and word[i + 1] == second:
																												new_word.append(first + second  )
																												i += 2
																						else:
																												new_word.append(word[i]  )
																												i += 1
																UpperCAmelCase__       						=		tuple(_lowercase  )
																UpperCAmelCase__       						=		new_word
																if len(_lowercase  ) == 1:
																						break
																else:
																						UpperCAmelCase__       						=		get_pairs(_lowercase  )
										UpperCAmelCase__       						=		""" """.join(_lowercase  )
										UpperCAmelCase__       						=		word
										return word

				def 							A			(  self							:     Optional[int]							,     _lowercase							:     Tuple  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		[]
										for token in re.findall(self.pat							,     _lowercase  ):
																UpperCAmelCase__       						=		"""""".join(
																    self.byte_encoder[b] for b in token.encode("""utf-8"""  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowercase  ).split(""" """  )  )
										return bpe_tokens

				def 							A			(  self							:     Any							,     _lowercase							:     str  ):


										'''simple docstring'''
										return self.encoder.get(_lowercase							,     self.encoder.get(self.unk_token  )  )

				def 							A			(  self							:     List[str]							,     _lowercase							:     List[str]  ):


										'''simple docstring'''
										return self.decoder.get(_lowercase  )

				def 							A			(  self							:     List[Any]							,     _lowercase							:     Optional[Any]  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		"""""".join(_lowercase  )
										UpperCAmelCase__       						=		bytearray([self.byte_decoder[c] for c in text]  ).decode("""utf-8"""							,     errors=self.errors  )
										return text

				def 							A			(  self							:     Optional[Any]							,     _lowercase							:     str							,     _lowercase							:     Optional[str] = None  ):


										'''simple docstring'''
										if not os.path.isdir(_lowercase  ):
																logger.error(f'Vocabulary path ({save_directory}) should be a directory'  )
																return
										UpperCAmelCase__       						=		os.path.join(
										    _lowercase							,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
										UpperCAmelCase__       						=		os.path.join(
										    _lowercase							,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

										with open(_lowercase							,     """w"""							,     encoding="""utf-8"""  ) as f:
																f.write(json.dumps(self.encoder							,     indent=2							,     sort_keys=_lowercase							,     ensure_ascii=_lowercase  ) + """\n"""  )

										UpperCAmelCase__       						=		0
										with open(_lowercase							,     """w"""							,     encoding="""utf-8"""  ) as writer:
																writer.write("""#version: 0.2\n"""  )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,     key=lambda _lowercase  : kv[1]  ):
																						if index != token_index:
																												logger.warning(
																												    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																												    """ Please check that the tokenizer is not corrupted!"""  )
																												UpperCAmelCase__       						=		token_index
																						writer.write(""" """.join(_lowercase  ) + """\n"""  )
																						index += 1

										return vocab_file, merge_file

				def 							A			(  self							:     int							,     _lowercase							:     List[int]							,     _lowercase							:     Optional[List[int]] = None  ):


										'''simple docstring'''
										if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										UpperCAmelCase__       						=		[self.cls_token_id]
										UpperCAmelCase__       						=		[self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep

				def 							A			(  self							:     Optional[int]							,     _lowercase							:     List[int]							,     _lowercase							:     Optional[List[int]] = None							,     _lowercase							:     bool = False  ):


										'''simple docstring'''
										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_lowercase							,     token_ids_a=_lowercase							,     already_has_special_tokens=_lowercase  )

										if token_ids_a is None:
																return [1] + ([0] * len(_lowercase  )) + [1]
										return [1] + ([0] * len(_lowercase  )) + [1, 1] + ([0] * len(_lowercase  )) + [1]

				def 							A			(  self							:     Optional[int]							,     _lowercase							:     List[int]							,     _lowercase							:     Optional[List[int]] = None  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		[self.sep_token_id]
										UpperCAmelCase__       						=		[self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep  ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]

				def 							A			(  self							:     List[Any]							,     _lowercase							:     Tuple							,     _lowercase							:     Union[str, Any]=False							,     **_lowercase							:     int  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		kwargs.pop("""add_prefix_space"""							,     self.add_prefix_space  )
										if (is_split_into_words or add_prefix_space) and (len(_lowercase  ) > 0 and not text[0].isspace()):
																UpperCAmelCase__       						=		""" """ + text
										return (text, kwargs)


"""simple docstring"""







import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


UpperCamelCase_							:							Optional[int]				 =		logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
UpperCamelCase_							:							Optional[Any]				 =		direct_transformers_import(Path(__file__).parent)


UpperCamelCase_							:							Optional[int]				 =		{
    """AutoTokenizer""": """PreTrainedTokenizerBase""",
    """AutoFeatureExtractor""": """FeatureExtractionMixin""",
    """AutoImageProcessor""": """ImageProcessingMixin""",
}


class       _lowercase		( __lowerCamelCase	):
				lowerCamelCase__							=							['feature_extractor', 'tokenizer']
				# Names need to be attr_class for attr in attributes
				lowerCamelCase__							=							None
				lowerCamelCase__							=							None
				lowerCamelCase__							=							None

				def __init__(  self							:     Any							,     *_lowercase							:     Optional[int]							,     **_lowercase							:     int  ):


										'''simple docstring'''
										# Sanitize args and kwargs
										for key in kwargs:
																if key not in self.attributes:
																						raise TypeError(f'Unexpected keyword argument {key}.'  )
										for arg, attribute_name in zip(_lowercase							,     self.attributes  ):
																if attribute_name in kwargs:
																						raise TypeError(f'Got multiple values for argument {attribute_name}.'  )
																else:
																						UpperCAmelCase__       						=		arg

										if len(_lowercase  ) != len(self.attributes  ):
																raise ValueError(
																    f'This processor requires {len(self.attributes  )} arguments: {", ".join(self.attributes  )}. Got '
																    f'{len(_lowercase  )} arguments instead.'  )

										# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
										for attribute_name, arg in kwargs.items():
																UpperCAmelCase__       						=		getattr(self							,     f'{attribute_name}_class'  )
																# Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
																UpperCAmelCase__       						=		AUTO_TO_BASE_CLASS_MAPPING.get(_lowercase							,     _lowercase  )
																if isinstance(_lowercase							,     _lowercase  ):
																						UpperCAmelCase__       						=		tuple(getattr(_lowercase							,     _lowercase  ) for n in class_name if n is not None  )
																else:
																						UpperCAmelCase__       						=		getattr(_lowercase							,     _lowercase  )

																if not isinstance(_lowercase							,     _lowercase  ):
																						raise ValueError(
																						    f'Received a {type(_lowercase  ).__name__} for argument {attribute_name}, but a {class_name} was expected.'  )

																setattr(self							,     _lowercase							,     _lowercase  )

				def __repr__(  self							:     Tuple  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		[f'- {name}: {repr(getattr(self							,     _lowercase  )  )}' for name in self.attributes]
										UpperCAmelCase__       						=		"""\n""".join(_lowercase  )
										return f'{self.__class__.__name__}:\n{attributes_repr}'

				def 							A			(  self							:     Tuple							,     _lowercase							:     Dict							,     _lowercase							:     bool = False							,     **_lowercase							:     List[str]  ):


										'''simple docstring'''
										os.makedirs(_lowercase							,     exist_ok=_lowercase  )

										if push_to_hub:
																UpperCAmelCase__       						=		kwargs.pop("""commit_message"""							,     _lowercase  )
																UpperCAmelCase__       						=		kwargs.pop("""repo_id"""							,     save_directory.split(os.path.sep  )[-1]  )
																UpperCAmelCase__       						=		self._create_repo(_lowercase							,     **_lowercase  )
																UpperCAmelCase__       						=		self._get_files_timestamps(_lowercase  )
										# If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
										# loaded from the Hub.
										if self._auto_class is not None:
																UpperCAmelCase__       						=		[getattr(self							,     _lowercase  ) for attribute_name in self.attributes]
																UpperCAmelCase__       						=		[(a.init_kwargs if isinstance(_lowercase							,     _lowercase  ) else a) for a in attrs]
																custom_object_save(self							,     _lowercase							,     config=_lowercase  )

										for attribute_name in self.attributes:
																UpperCAmelCase__       						=		getattr(self							,     _lowercase  )
																# Include the processor class in the attribute config so this processor can then be reloaded with the
																# `AutoProcessor` API.
																if hasattr(_lowercase							,     """_set_processor_class"""  ):
																						attribute._set_processor_class(self.__class__.__name__  )
																attribute.save_pretrained(_lowercase  )

										if self._auto_class is not None:
																# We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
																for attribute_name in self.attributes:
																						UpperCAmelCase__       						=		getattr(self							,     _lowercase  )
																						if isinstance(_lowercase							,     _lowercase  ):
																												del attribute.init_kwargs["auto_map"]

										if push_to_hub:
																self._upload_modified_files(
																    _lowercase							,     _lowercase							,     _lowercase							,     commit_message=_lowercase							,     token=kwargs.get("""use_auth_token"""  )							,     )

				@classmethod
				def 							A			(  cls							:     Dict							,     _lowercase							:     Union[str, os.PathLike]							,     _lowercase							:     Optional[Union[str, os.PathLike]] = None							,     _lowercase							:     bool = False							,     _lowercase							:     bool = False							,     _lowercase							:     Optional[Union[str, bool]] = None							,     _lowercase							:     str = "main"							,     **_lowercase							:     Dict							,     ):


										'''simple docstring'''
										UpperCAmelCase__       						=		cache_dir
										UpperCAmelCase__       						=		force_download
										UpperCAmelCase__       						=		local_files_only
										UpperCAmelCase__       						=		revision

										UpperCAmelCase__       						=		kwargs.pop("""use_auth_token"""							,     _lowercase  )
										if use_auth_token is not None:
																warnings.warn(
																    """The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."""							,     _lowercase  )
																if token is not None:
																						raise ValueError(
																						    """`token` and `use_auth_token` are both specified. Please set only the argument `token`."""  )
																UpperCAmelCase__       						=		use_auth_token

										if token is not None:
																# change to `token` in a follow-up PR
																UpperCAmelCase__       						=		token

										UpperCAmelCase__       						=		cls._get_arguments_from_pretrained(_lowercase							,     **_lowercase  )
										return cls(*_lowercase  )

				@classmethod
				def 							A			(  cls							:     List[Any]							,     _lowercase							:     Dict="AutoProcessor"  ):


										'''simple docstring'''
										if not isinstance(_lowercase							,     _lowercase  ):
																UpperCAmelCase__       						=		auto_class.__name__

										import transformers.models.auto as auto_module

										if not hasattr(_lowercase							,     _lowercase  ):
																raise ValueError(f'{auto_class} is not a valid auto class.'  )

										UpperCAmelCase__       						=		auto_class

				@classmethod
				def 							A			(  cls							:     Any							,     _lowercase							:     Tuple							,     **_lowercase							:     List[Any]  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		[]
										for attribute_name in cls.attributes:
																UpperCAmelCase__       						=		getattr(cls							,     f'{attribute_name}_class'  )
																if isinstance(_lowercase							,     _lowercase  ):
																						UpperCAmelCase__       						=		tuple(getattr(_lowercase							,     _lowercase  ) if n is not None else None for n in class_name  )
																						UpperCAmelCase__       						=		kwargs.get("""use_fast"""							,     _lowercase  )
																						if use_fast and classes[1] is not None:
																												UpperCAmelCase__       						=		classes[1]
																						else:
																												UpperCAmelCase__       						=		classes[0]
																else:
																						UpperCAmelCase__       						=		getattr(_lowercase							,     _lowercase  )

																args.append(attribute_class.from_pretrained(_lowercase							,     **_lowercase  )  )
										return args

				@property
				def 							A			(  self							:     int  ):


										'''simple docstring'''
										UpperCAmelCase__       						=		getattr(self							,     self.attributes[0]  )
										return getattr(_lowercase							,     """model_input_names"""							,     _lowercase  )
UpperCamelCase_							:							List[Any]				 =		copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
						UpperCamelCase_							:							Dict				 =		ProcessorMixin.push_to_hub.__doc__.format(
						    object="""processor""", object_class="""AutoProcessor""", object_files="""processor files"""
						)

