

"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE							:	Dict										=			logging.get_logger(__name__)


_SCREAMING_SNAKE_CASE							:	str										=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
_SCREAMING_SNAKE_CASE							:	List[Any]										=			{
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
}

_SCREAMING_SNAKE_CASE							:	str										=			{
    """RUCAIBox/mvp""": 1_0_2_4,
}



@lru_cache()
def 							_lowerCAmelCase   (    ):


					'''simple docstring'''






					UpperCamelCase__		:     Any								=(
					    list(range(ord('''!'''					)   ,				ord('''~'''					) + 1					)					) + list(range(ord('''¡'''					)   ,				ord('''¬'''					) + 1					)					) + list(range(ord('''®'''					)   ,				ord('''ÿ'''					) + 1					)					)
					)
					UpperCamelCase__		:     int								=bs[:]
					UpperCamelCase__		:     List[Any]								=0
					for b in range(2**8					):
										if b not in bs:
															bs.append(A_					)
															cs.append(2**8 + n					)
															n += 1
					UpperCamelCase__		:     Any								=[chr(A_					) for n in cs]
					return dict(zip(A_   ,				A_					)					)



def 							_lowerCAmelCase   (    UpperCAmelCase  :				Dict					):


					'''simple docstring'''






					UpperCamelCase__		:     Tuple								=set()
					UpperCamelCase__		:     Any								=word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)					)
										UpperCamelCase__		:     Tuple								=char
					return pairs







class      __a (   _a     ):

			"""simple docstring"""

			SCREAMING_SNAKE_CASE_      =    VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE_      =    PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE_      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE_      =    ["""input_ids""", """attention_mask"""]






			def __init__(				self  :       Tuple    ,	lowercase_  :       Dict    ,	lowercase_  :       Dict    ,	lowercase_  :       str="replace"    ,	lowercase_  :       Tuple="<s>"    ,	lowercase_  :       List[str]="</s>"    ,	lowercase_  :       Optional[int]="</s>"    ,	lowercase_  :       Tuple="<s>"    ,	lowercase_  :       List[Any]="<unk>"    ,	lowercase_  :       Tuple="<pad>"    ,	lowercase_  :       Dict="<mask>"    ,	lowercase_  :       str=False    ,	**lowercase_  :       List[Any]    ,	):
								UpperCamelCase__		:     Any								=AddedToken(__UpperCAmelCase    ,	lstrip=__UpperCAmelCase    ,	rstrip=__UpperCAmelCase    ) if isinstance(__UpperCAmelCase    ,	__UpperCAmelCase    ) else bos_token
								UpperCamelCase__		:     str								=AddedToken(__UpperCAmelCase    ,	lstrip=__UpperCAmelCase    ,	rstrip=__UpperCAmelCase    ) if isinstance(__UpperCAmelCase    ,	__UpperCAmelCase    ) else eos_token
								UpperCamelCase__		:     Union[str, Any]								=AddedToken(__UpperCAmelCase    ,	lstrip=__UpperCAmelCase    ,	rstrip=__UpperCAmelCase    ) if isinstance(__UpperCAmelCase    ,	__UpperCAmelCase    ) else sep_token
								UpperCamelCase__		:     List[str]								=AddedToken(__UpperCAmelCase    ,	lstrip=__UpperCAmelCase    ,	rstrip=__UpperCAmelCase    ) if isinstance(__UpperCAmelCase    ,	__UpperCAmelCase    ) else cls_token
								UpperCamelCase__		:     List[Any]								=AddedToken(__UpperCAmelCase    ,	lstrip=__UpperCAmelCase    ,	rstrip=__UpperCAmelCase    ) if isinstance(__UpperCAmelCase    ,	__UpperCAmelCase    ) else unk_token
								UpperCamelCase__		:     List[Any]								=AddedToken(__UpperCAmelCase    ,	lstrip=__UpperCAmelCase    ,	rstrip=__UpperCAmelCase    ) if isinstance(__UpperCAmelCase    ,	__UpperCAmelCase    ) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								UpperCamelCase__		:     Optional[int]								=AddedToken(__UpperCAmelCase    ,	lstrip=__UpperCAmelCase    ,	rstrip=__UpperCAmelCase    ) if isinstance(__UpperCAmelCase    ,	__UpperCAmelCase    ) else mask_token

								super().__init__(
								    errors=__UpperCAmelCase    ,	bos_token=__UpperCAmelCase    ,	eos_token=__UpperCAmelCase    ,	unk_token=__UpperCAmelCase    ,	sep_token=__UpperCAmelCase    ,	cls_token=__UpperCAmelCase    ,	pad_token=__UpperCAmelCase    ,	mask_token=__UpperCAmelCase    ,	add_prefix_space=__UpperCAmelCase    ,	**__UpperCAmelCase    ,	)
								with open(__UpperCAmelCase    ,	encoding='''utf-8'''    ) as vocab_handle:
													UpperCamelCase__		:     str								=json.load(__UpperCAmelCase    )
								UpperCamelCase__		:     Dict								={v: k for k, v in self.encoder.items()}
								UpperCamelCase__		:     str								=errors  # how to handle errors in decoding
								UpperCamelCase__		:     int								=bytes_to_unicode()
								UpperCamelCase__		:     Union[str, Any]								={v: k for k, v in self.byte_encoder.items()}
								with open(__UpperCAmelCase    ,	encoding='''utf-8'''    ) as merges_handle:
													UpperCamelCase__		:     List[str]								=merges_handle.read().split('''\n'''    )[1:-1]
								UpperCamelCase__		:     int								=[tuple(merge.split()    ) for merge in bpe_merges]
								UpperCamelCase__		:     int								=dict(zip(__UpperCAmelCase    ,	range(len(__UpperCAmelCase    )    )    )    )
								UpperCamelCase__		:     Any								={}
								UpperCamelCase__		:     int								=add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								UpperCamelCase__		:     Dict								=re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''    )






			@property
			def  _lowerCAmelCase      (				self  :       Any    ):
								return len(self.encoder    )






			def  _lowerCAmelCase      (				self  :       Optional[int]    ):
								return dict(self.encoder    ,	**self.added_tokens_encoder    )






			def  _lowerCAmelCase      (				self  :       Tuple    ,	lowercase_  :       int    ):
								if token in self.cache:
													return self.cache[token]
								UpperCamelCase__		:     Any								=tuple(__UpperCAmelCase    )
								UpperCamelCase__		:     Any								=get_pairs(__UpperCAmelCase    )

								if not pairs:
													return token

								while True:
													UpperCamelCase__		:     Optional[int]								=min(__UpperCAmelCase    ,	key=lambda lowercase_    : self.bpe_ranks.get(__UpperCAmelCase    ,	float('''inf'''    )    )    )
													if bigram not in self.bpe_ranks:
																		break
													UpperCamelCase__		:     Union[str, Any]								=bigram
													UpperCamelCase__		:     Dict								=[]
													UpperCamelCase__		:     Optional[int]								=0
													while i < len(__UpperCAmelCase    ):
																		try:
																							UpperCamelCase__		:     Union[str, Any]								=word.index(__UpperCAmelCase    ,	__UpperCAmelCase    )
																		except ValueError:
																							new_word.extend(word[i:]    )
																							break
																		else:
																							new_word.extend(word[i:j]    )
																							UpperCamelCase__		:     List[str]								=j

																		if word[i] == first and i < len(__UpperCAmelCase    ) - 1 and word[i + 1] == second:
																							new_word.append(first + second    )
																							i += 2
																		else:
																							new_word.append(word[i]    )
																							i += 1
													UpperCamelCase__		:     Tuple								=tuple(__UpperCAmelCase    )
													UpperCamelCase__		:     List[Any]								=new_word
													if len(__UpperCAmelCase    ) == 1:
																		break
													else:
																		UpperCamelCase__		:     str								=get_pairs(__UpperCAmelCase    )
								UpperCamelCase__		:     str								=" ".join(__UpperCAmelCase    )
								UpperCamelCase__		:     Dict								=word
								return word






			def  _lowerCAmelCase      (				self  :       List[Any]    ,	lowercase_  :       List[str]    ):
								UpperCamelCase__		:     List[Any]								=[]
								for token in re.findall(self.pat    ,	__UpperCAmelCase    ):
													UpperCamelCase__		:     List[Any]								="".join(
													    self.byte_encoder[b] for b in token.encode('''utf-8'''    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
													bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__UpperCAmelCase    ).split(''' '''    )    )
								return bpe_tokens






			def  _lowerCAmelCase      (				self  :       Dict    ,	lowercase_  :       Union[str, Any]    ):
								return self.encoder.get(__UpperCAmelCase    ,	self.encoder.get(self.unk_token    )    )






			def  _lowerCAmelCase      (				self  :       Tuple    ,	lowercase_  :       Union[str, Any]    ):
								return self.decoder.get(__UpperCAmelCase    )






			def  _lowerCAmelCase      (				self  :       int    ,	lowercase_  :       Optional[Any]    ):
								UpperCamelCase__		:     List[Any]								="".join(__UpperCAmelCase    )
								UpperCamelCase__		:     Tuple								=bytearray([self.byte_decoder[c] for c in text]    ).decode('''utf-8'''    ,	errors=self.errors    )
								return text






			def  _lowerCAmelCase      (				self  :       Dict    ,	lowercase_  :       str    ,	lowercase_  :       Optional[str] = None    ):
								if not os.path.isdir(__UpperCAmelCase    ):
													logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
													return
								UpperCamelCase__		:     List[Any]								=os.path.join(
								    __UpperCAmelCase    ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
								UpperCamelCase__		:     str								=os.path.join(
								    __UpperCAmelCase    ,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']    )

								with open(__UpperCAmelCase    ,	'''w'''    ,	encoding='''utf-8'''    ) as f:
													f.write(json.dumps(self.encoder    ,	indent=2    ,	sort_keys=__UpperCAmelCase    ,	ensure_ascii=__UpperCAmelCase    ) + '''\n'''    )

								UpperCamelCase__		:     Any								=0
								with open(__UpperCAmelCase    ,	'''w'''    ,	encoding='''utf-8'''    ) as writer:
													writer.write('''#version: 0.2\n'''    )
													for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,	key=lambda lowercase_    : kv[1]    ):
																		if index != token_index:
																							logger.warning(
																							    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																							    ''' Please check that the tokenizer is not corrupted!'''    )
																							UpperCamelCase__		:     Optional[int]								=token_index
																		writer.write(''' '''.join(__UpperCAmelCase    ) + '''\n'''    )
																		index += 1

								return vocab_file, merge_file






			def  _lowerCAmelCase      (				self  :       List[str]    ,	lowercase_  :       List[int]    ,	lowercase_  :       Optional[List[int]] = None    ):
								if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								UpperCamelCase__		:     Any								=[self.cls_token_id]
								UpperCamelCase__		:     Any								=[self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep






			def  _lowerCAmelCase      (				self  :       Optional[Any]    ,	lowercase_  :       List[int]    ,	lowercase_  :       Optional[List[int]] = None    ,	lowercase_  :       bool = False    ):
								if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=__UpperCAmelCase    ,	token_ids_a=__UpperCAmelCase    ,	already_has_special_tokens=__UpperCAmelCase    )

								if token_ids_a is None:
													return [1] + ([0] * len(__UpperCAmelCase    )) + [1]
								return [1] + ([0] * len(__UpperCAmelCase    )) + [1, 1] + ([0] * len(__UpperCAmelCase    )) + [1]






			def  _lowerCAmelCase      (				self  :       str    ,	lowercase_  :       List[int]    ,	lowercase_  :       Optional[List[int]] = None    ):
								UpperCamelCase__		:     List[str]								=[self.sep_token_id]
								UpperCamelCase__		:     Tuple								=[self.cls_token_id]

								if token_ids_a is None:
													return len(cls + token_ids_a + sep    ) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]






			def  _lowerCAmelCase      (				self  :       Optional[int]    ,	lowercase_  :       Optional[int]    ,	lowercase_  :       Union[str, Any]=False    ,	**lowercase_  :       List[str]    ):
								UpperCamelCase__		:     int								=kwargs.pop('''add_prefix_space'''    ,	self.add_prefix_space    )
								if (is_split_into_words or add_prefix_space) and (len(__UpperCAmelCase    ) > 0 and not text[0].isspace()):
													UpperCamelCase__		:     Optional[Any]								=" " + text
								return (text, kwargs)






"""simple docstring"""







def 	lowercase     (							A_ = 100    ,  A_ = 100			)-> int:
  '''simple docstring'''







  return max(
      sum(int(A_			) for x in str(base**power			)			)
      for base in range(A_			)
      for power in range(A_			)			)


# Tests
if __name__ == "__main__":
 import doctest

 doctest.testmod()






