



import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase	:   List[Any]    	=logging.get_logger(__name__)

_lowercase	:   List[str]    	={
    'src_vocab_file': 'vocab-src.json',
    'tgt_vocab_file': 'vocab-tgt.json',
    'merges_file': 'merges.txt',
}

_lowercase	:   Tuple    	={
    'src_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json'
    },
    'tgt_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json'
    },
    'merges_file': {'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt'},
}

_lowercase	:   Optional[int]    	={'stas/tiny-wmt19-en-de': 1024}
_lowercase	:   Union[str, Any]    	={
    'stas/tiny-wmt19-en-de': {
        'langs': ['en', 'de'],
        'model_max_length': 1024,
        'special_tokens_map_file': None,
        'full_tokenizer_file': None,
    }
}

def 			__A  (						_lowercase       :     int)							-> Optional[int]:







			"""simple docstring"""
			a__     :		List[Any]		 =						set()
			a__     :		Any		 =						word[0]
			for char in word[1:]:
						pairs.add((prev_char, char))
						a__     :		Union[str, Any]		 =						char
			return pairs

def 			__A  (						_lowercase       :     List[Any])							-> List[str]:







			"""simple docstring"""
			a__     :		Union[str, Any]		 =						text.replace("""，"""		,     """,""")
			a__     :		str		 =						re.sub(R"""。\s*"""		,     """. """		,     __A)
			a__     :		Optional[int]		 =						text.replace("""、"""		,     """,""")
			a__     :		Dict		 =						text.replace("""”"""		,     """\"""")
			a__     :		List[Any]		 =						text.replace("""“"""		,     """\"""")
			a__     :		Dict		 =						text.replace("""∶"""		,     """:""")
			a__     :		int		 =						text.replace("""："""		,     """:""")
			a__     :		Any		 =						text.replace("""？"""		,     """?""")
			a__     :		Tuple		 =						text.replace("""《"""		,     """\"""")
			a__     :		Optional[Any]		 =						text.replace("""》"""		,     """\"""")
			a__     :		Optional[int]		 =						text.replace("""）"""		,     """)""")
			a__     :		Union[str, Any]		 =						text.replace("""！"""		,     """!""")
			a__     :		int		 =						text.replace("""（"""		,     """(""")
			a__     :		Union[str, Any]		 =						text.replace("""；"""		,     """;""")
			a__     :		Dict		 =						text.replace("""１"""		,     """1""")
			a__     :		Union[str, Any]		 =						text.replace("""」"""		,     """\"""")
			a__     :		Union[str, Any]		 =						text.replace("""「"""		,     """\"""")
			a__     :		Any		 =						text.replace("""０"""		,     """0""")
			a__     :		str		 =						text.replace("""３"""		,     """3""")
			a__     :		Dict		 =						text.replace("""２"""		,     """2""")
			a__     :		Optional[Any]		 =						text.replace("""５"""		,     """5""")
			a__     :		Optional[int]		 =						text.replace("""６"""		,     """6""")
			a__     :		str		 =						text.replace("""９"""		,     """9""")
			a__     :		Union[str, Any]		 =						text.replace("""７"""		,     """7""")
			a__     :		Tuple		 =						text.replace("""８"""		,     """8""")
			a__     :		str		 =						text.replace("""４"""		,     """4""")
			a__     :		Any		 =						re.sub(R"""．\s*"""		,     """. """		,     __A)
			a__     :		Any		 =						text.replace("""～"""		,     """~""")
			a__     :		Union[str, Any]		 =						text.replace("""’"""		,     """'""")
			a__     :		List[str]		 =						text.replace("""…"""		,     """...""")
			a__     :		List[str]		 =						text.replace("""━"""		,     """-""")
			a__     :		int		 =						text.replace("""〈"""		,     """<""")
			a__     :		Optional[Any]		 =						text.replace("""〉"""		,     """>""")
			a__     :		Dict		 =						text.replace("""【"""		,     """[""")
			a__     :		List[Any]		 =						text.replace("""】"""		,     """]""")
			a__     :		str		 =						text.replace("""％"""		,     """%""")
			return text

def 			__A  (						_lowercase       :     Optional[Any])							-> List[str]:







			"""simple docstring"""
			a__     :		Optional[int]		 =						[]
			for char in text:
						a__     :		Dict		 =						unicodedata.category(__A)
						if cat.startswith("""C"""):
									continue
						output.append(__A)
			return "".join(__A)





class      snake_case__       (UpperCAmelCase_	):







						"""simple docstring"""




						__lowerCAmelCase  :Optional[int]			      =  VOCAB_FILES_NAMES
						__lowerCAmelCase  :Any			      =  PRETRAINED_VOCAB_FILES_MAP
						__lowerCAmelCase  :Tuple			      =  PRETRAINED_INIT_CONFIGURATION
						__lowerCAmelCase  :Tuple			      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__lowerCAmelCase  :Optional[int]			      =  ["input_ids", "attention_mask"]

						def __init__(     self      ,     __lowercase=None      ,     __lowercase=None      ,     __lowercase=None      ,     __lowercase=None      ,     __lowercase=False      ,     __lowercase="<unk>"      ,     __lowercase="<s>"      ,     __lowercase="</s>"      ,     __lowercase="<pad>"      ,     **__lowercase      ,     )      ->   Any:
									"""simple docstring"""




									super().__init__(
									    langs=__lowercase      ,     src_vocab_file=__lowercase      ,     tgt_vocab_file=__lowercase      ,     merges_file=__lowercase      ,     do_lower_case=__lowercase      ,     unk_token=__lowercase      ,     bos_token=__lowercase      ,     sep_token=__lowercase      ,     pad_token=__lowercase      ,     **__lowercase      ,     )

									try:
												import sacremoses
									except ImportError:
												raise ImportError(
												    """You need to install sacremoses to use XLMTokenizer. """
												    """See https://pypi.org/project/sacremoses/ for installation."""						)

									a__     :		Any		 =						sacremoses

									a__     :		Union[str, Any]		 =						src_vocab_file
									a__     :		Tuple		 =						tgt_vocab_file
									a__     :		List[str]		 =						merges_file
									a__     :		Dict		 =						do_lower_case

									# cache of sm.MosesPunctNormalizer instance
									a__     :		Dict		 =						{}
									# cache of sm.MosesTokenizer instance
									a__     :		Dict		 =						{}
									a__     :		Dict		 =						{}

									if langs and len(__lowercase						) == 2:
												a__ ,		a__     :		Optional[Any]		 =						langs
									else:
												raise ValueError(
												    F'''arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '''
												    """Usually that means that tokenizer can't find a mapping for the given model path """
												    """in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer."""						)

									with open(__lowercase      ,     encoding="""utf-8"""						) as src_vocab_handle:
												a__     :		List[str]		 =						json.load(__lowercase						)
									with open(__lowercase      ,     encoding="""utf-8"""						) as tgt_vocab_handle:
												a__     :		List[Any]		 =						json.load(__lowercase						)
												a__     :		Optional[int]		 =						{v: k for k, v in tgt_vocab.items()}
									with open(__lowercase      ,     encoding="""utf-8"""						) as merges_handle:
												a__     :		Optional[int]		 =						merges_handle.read().split("""\n"""						)[:-1]
									a__     :		Any		 =						[tuple(merge.split()[:2]						) for merge in merges]
									a__     :		Optional[Any]		 =						dict(zip(__lowercase      ,     range(len(__lowercase						)						)						)						)
									a__     :		Tuple		 =						{}

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Dict[str, int]:
									"""simple docstring"""




									return self.get_src_vocab()

						@property
						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   int:
									"""simple docstring"""




									return self.src_vocab_size

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase						)      ->   Dict:
									"""simple docstring"""




									if lang not in self.cache_moses_punct_normalizer:
												a__     :		Tuple		 =						self.sm.MosesPunctNormalizer(lang=__lowercase						)
												a__     :		Union[str, Any]		 =						punct_normalizer
									return self.cache_moses_punct_normalizer[lang].normalize(__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase						)      ->   Optional[int]:
									"""simple docstring"""




									if lang not in self.cache_moses_tokenizer:
												a__     :		Any		 =						self.sm.MosesTokenizer(lang=__lowercase						)
												a__     :		int		 =						moses_tokenizer
									return self.cache_moses_tokenizer[lang].tokenize(
									    __lowercase      ,     aggressive_dash_splits=__lowercase      ,     return_str=__lowercase      ,     escape=__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase						)      ->   Union[str, Any]:
									"""simple docstring"""




									if lang not in self.cache_moses_detokenizer:
												a__     :		List[Any]		 =						self.sm.MosesDetokenizer(lang=__lowercase						)
												a__     :		List[Any]		 =						moses_detokenizer
									return self.cache_moses_detokenizer[lang].detokenize(__lowercase						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase						)      ->   List[Any]:
									"""simple docstring"""




									a__     :		str		 =						replace_unicode_punct(__lowercase						)
									a__     :		List[Any]		 =						self.moses_punct_norm(__lowercase      ,     __lowercase						)
									a__     :		Optional[Any]		 =						remove_non_printing_char(__lowercase						)
									return text

						@property
						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   List[str]:
									"""simple docstring"""




									return len(self.encoder						)

						@property
						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Optional[int]:
									"""simple docstring"""




									return len(self.decoder						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Any:
									"""simple docstring"""




									return dict(self.encoder      ,     **self.added_tokens_encoder						)

						def 							SCREAMING_SNAKE_CASE__(     self						)      ->   Any:
									"""simple docstring"""




									return dict(self.decoder      ,     **self.added_tokens_decoder						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase						)      ->   Dict:
									"""simple docstring"""




									a__     :		Optional[int]		 =						tuple(token[:-1]						) + (token[-1] + """</w>""",)
									if token in self.cache:
												return self.cache[token]
									a__     :		Tuple		 =						get_pairs(__lowercase						)

									if not pairs:
												return token + "</w>"

									while True:
												a__     :		Any		 =						min(__lowercase      ,     key=lambda __lowercase						: self.bpe_ranks.get(__lowercase      ,     float("""inf"""						)						)						)
												if bigram not in self.bpe_ranks:
															break
												a__ ,		a__     :		Optional[int]		 =						bigram
												a__     :		Optional[Any]		 =						[]
												a__     :		List[Any]		 =						0
												while i < len(__lowercase						):
															try:
																		a__     :		List[str]		 =						word.index(__lowercase      ,     __lowercase						)
															except ValueError:
																		new_word.extend(word[i:]						)
																		break
															else:
																		new_word.extend(word[i:j]						)
																		a__     :		Any		 =						j

															if word[i] == first and i < len(__lowercase						) - 1 and word[i + 1] == second:
																		new_word.append(first + second						)
																		i += 2
															else:
																		new_word.append(word[i]						)
																		i += 1
												a__     :		List[Any]		 =						tuple(__lowercase						)
												a__     :		Dict		 =						new_word
												if len(__lowercase						) == 1:
															break
												else:
															a__     :		Union[str, Any]		 =						get_pairs(__lowercase						)
									a__     :		Union[str, Any]		 =						""" """.join(__lowercase						)
									if word == "\n  </w>":
												a__     :		str		 =						"""\n</w>"""
									a__     :		List[str]		 =						word
									return word

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase="en"      ,     __lowercase=False						)      ->   Optional[int]:
									"""simple docstring"""




									a__     :		Any		 =						self.src_lang

									if self.do_lower_case:
												a__     :		List[str]		 =						text.lower()

									if bypass_tokenizer:
												a__     :		List[Any]		 =						text.split()
									else:
												a__     :		Optional[int]		 =						self.moses_pipeline(__lowercase      ,     lang=__lowercase						)
												a__     :		List[Any]		 =						self.moses_tokenize(__lowercase      ,     lang=__lowercase						)

									a__     :		Tuple		 =						[]
									for token in text:
												if token:
															split_tokens.extend(list(self.bpe(__lowercase						).split(""" """						)						)						)

									return split_tokens

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase						)      ->   Any:
									"""simple docstring"""




									return self.encoder.get(__lowercase      ,     self.encoder.get(self.unk_token						)						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase						)      ->   str:
									"""simple docstring"""




									return self.decoder.get(__lowercase      ,     self.unk_token						)

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase						)      ->   Tuple:
									"""simple docstring"""




									a__     :		Union[str, Any]		 =						[t.replace(""" """      ,     """"""						).replace("""</w>"""      ,     """ """						) for t in tokens]
									a__     :		Union[str, Any]		 =						"""""".join(__lowercase						).split()
									# detokenize
									a__     :		Optional[Any]		 =						self.moses_detokenize(__lowercase      ,     self.tgt_lang						)
									return text

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase = None						)      ->   List[int]:
									"""simple docstring"""




									a__     :		Optional[Any]		 =						[self.sep_token_id]

									# no bos used in fairseq
									if token_ids_a is None:
												return token_ids_a + sep
									return token_ids_a + sep + token_ids_a + sep

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase = None      ,     __lowercase = False						)      ->   List[int]:
									"""simple docstring"""




									if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__lowercase      ,     token_ids_a=__lowercase      ,     already_has_special_tokens=__lowercase						)
									# no bos used in fairseq
									if token_ids_a is not None:
												return ([0] * len(__lowercase						)) + [1] + ([0] * len(__lowercase						)) + [1]
									return ([0] * len(__lowercase						)) + [1]

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase = None						)      ->   List[int]:
									"""simple docstring"""




									a__     :		Dict		 =						[self.sep_token_id]

									# no bos used in fairseq
									if token_ids_a is None:
												return len(token_ids_a + sep						) * [0]
									return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]

						def 							SCREAMING_SNAKE_CASE__(     self      ,     __lowercase      ,     __lowercase = None						)      ->   Tuple[str]:
									"""simple docstring"""




									if not os.path.isdir(__lowercase						):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
												return

									a__     :		Union[str, Any]		 =						os.path.join(
									    __lowercase      ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""src_vocab_file"""]						)
									a__     :		str		 =						os.path.join(
									    __lowercase      ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""tgt_vocab_file"""]						)
									a__     :		Optional[Any]		 =						os.path.join(
									    __lowercase      ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

									with open(__lowercase      ,     """w"""      ,     encoding="""utf-8"""						) as f:
												f.write(json.dumps(self.encoder      ,     indent=2      ,     sort_keys=__lowercase      ,     ensure_ascii=__lowercase						) + """\n"""						)

									with open(__lowercase      ,     """w"""      ,     encoding="""utf-8"""						) as f:
												a__     :		Union[str, Any]		 =						{v: k for k, v in self.decoder.items()}
												f.write(json.dumps(__lowercase      ,     indent=2      ,     sort_keys=__lowercase      ,     ensure_ascii=__lowercase						) + """\n"""						)

									a__     :		List[str]		 =						0
									with open(__lowercase      ,     """w"""      ,     encoding="""utf-8"""						) as writer:
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,     key=lambda __lowercase						: kv[1]						):
															if index != token_index:
																		logger.warning(
																		    F'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
																		    """ Please check that the tokenizer is not corrupted!"""						)
																		a__     :		List[str]		 =						token_index
															writer.write(""" """.join(__lowercase						) + """\n"""						)
															index += 1

									return src_vocab_file, tgt_vocab_file, merges_file

						def __getstate__(     self						)      ->   Union[str, Any]:
									"""simple docstring"""




									a__     :		Union[str, Any]		 =						self.__dict__.copy()
									a__     :		Union[str, Any]		 =						None
									return state

						def __setstate__(     self      ,     __lowercase						)      ->   List[Any]:
									"""simple docstring"""




									a__     :		Any		 =						d

									try:
												import sacremoses
									except ImportError:
												raise ImportError(
												    """You need to install sacremoses to use XLMTokenizer. """
												    """See https://pypi.org/project/sacremoses/ for installation."""						)

									a__     :		Dict		 =						sacremoses









import decimal

def 			lowerCAmelCase_  (						)							-> int:







			"""simple docstring"""
			a__     :		Optional[Any]		 =						0
			a__     :		Optional[int]		 =						decimal.Context(prec=105)
			for i in range(2		,     100):
						a__     :		int		 =						decimal.Decimal(_lowercase)
						a__     :		Optional[int]		 =						number.sqrt(_lowercase)
						if len(str(_lowercase)) > 1:
									answer += int(str(_lowercase)[0])
									a__     :		Tuple		 =						str(_lowercase)[2:101]
									answer += sum(int(_lowercase) for x in sqrt_number_str)
			return answer


if __name__ == "__main__":
		import doctest

		doctest.testmod()
		print(f'{solution() = }')




