

import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__							=				logging.get_logger(__name__)

A__							=				{
    "src_vocab_file": "vocab-src.json",
    "tgt_vocab_file": "vocab-tgt.json",
    "merges_file": "merges.txt",
}

A__							=				{
    "src_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"
    },
    "tgt_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"
    },
    "merges_file": {"stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"},
}

A__							=				{"stas/tiny-wmt19-en-de": 1024}
A__							=				{
    "stas/tiny-wmt19-en-de": {
        "langs": ["en", "de"],
        "model_max_length": 1024,
        "special_tokens_map_file": None,
        "full_tokenizer_file": None,
    }
}




def        _lowercase  (					a_					:       Dict					)					->						Tuple:




					'''simple docstring'''







					__magic_name__				=     set()
					__magic_name__				=     word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)					)
										__magic_name__				=     char
					return pairs




def        _lowercase  (					a_					:       Tuple					)					->						List[Any]:




					'''simple docstring'''







					__magic_name__				=     text.replace('，'       ,','					)
					__magic_name__				=     re.sub(R'。\s*'       ,'. '       ,a_					)
					__magic_name__				=     text.replace('、'       ,','					)
					__magic_name__				=     text.replace('”'       ,'"'					)
					__magic_name__				=     text.replace('“'       ,'"'					)
					__magic_name__				=     text.replace('∶'       ,':'					)
					__magic_name__				=     text.replace('：'       ,':'					)
					__magic_name__				=     text.replace('？'       ,'?'					)
					__magic_name__				=     text.replace('《'       ,'"'					)
					__magic_name__				=     text.replace('》'       ,'"'					)
					__magic_name__				=     text.replace('）'       ,')'					)
					__magic_name__				=     text.replace('！'       ,'!'					)
					__magic_name__				=     text.replace('（'       ,'('					)
					__magic_name__				=     text.replace('；'       ,';'					)
					__magic_name__				=     text.replace('１'       ,'1'					)
					__magic_name__				=     text.replace('」'       ,'"'					)
					__magic_name__				=     text.replace('「'       ,'"'					)
					__magic_name__				=     text.replace('０'       ,'0'					)
					__magic_name__				=     text.replace('３'       ,'3'					)
					__magic_name__				=     text.replace('２'       ,'2'					)
					__magic_name__				=     text.replace('５'       ,'5'					)
					__magic_name__				=     text.replace('６'       ,'6'					)
					__magic_name__				=     text.replace('９'       ,'9'					)
					__magic_name__				=     text.replace('７'       ,'7'					)
					__magic_name__				=     text.replace('８'       ,'8'					)
					__magic_name__				=     text.replace('４'       ,'4'					)
					__magic_name__				=     re.sub(R'．\s*'       ,'. '       ,a_					)
					__magic_name__				=     text.replace('～'       ,'~'					)
					__magic_name__				=     text.replace('’'       ,'\''					)
					__magic_name__				=     text.replace('…'       ,'...'					)
					__magic_name__				=     text.replace('━'       ,'-'					)
					__magic_name__				=     text.replace('〈'       ,'<'					)
					__magic_name__				=     text.replace('〉'       ,'>'					)
					__magic_name__				=     text.replace('【'       ,'['					)
					__magic_name__				=     text.replace('】'       ,']'					)
					__magic_name__				=     text.replace('％'       ,'%'					)
					return text




def        _lowercase  (					a_					:       Union[str, Any]					)					->						Dict:




					'''simple docstring'''







					__magic_name__				=     []
					for char in text:
										__magic_name__				=     unicodedata.category(a_					)
										if cat.startswith('C'					):
															continue
										output.append(a_					)
					return "".join(a_					)


class 					__UpperCamelCase					(       SCREAMING_SNAKE_CASE      ):
	_lowercase	: Dict												=	VOCAB_FILES_NAMES
	_lowercase	: int												=	PRETRAINED_VOCAB_FILES_MAP
	_lowercase	: Dict												=	PRETRAINED_INIT_CONFIGURATION
	_lowercase	: Dict												=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_lowercase	: Dict												=	["input_ids", "attention_mask"]







	def __init__( self:			Any	,			__UpperCamelCase:			Tuple=None	,			__UpperCamelCase:			int=None	,			__UpperCamelCase:			List[str]=None	,			__UpperCamelCase:			str=None	,			__UpperCamelCase:			str=False	,			__UpperCamelCase:			Union[str, Any]="<unk>"	,			__UpperCamelCase:			Any="<s>"	,			__UpperCamelCase:			List[Any]="</s>"	,			__UpperCamelCase:			List[Any]="<pad>"	,			**__UpperCamelCase:			Dict	,			):

						'''simple docstring'''





						super().__init__(
						    langs=__UpperCamelCase	,			src_vocab_file=__UpperCamelCase	,			tgt_vocab_file=__UpperCamelCase	,			merges_file=__UpperCamelCase	,			do_lower_case=__UpperCamelCase	,			unk_token=__UpperCamelCase	,			bos_token=__UpperCamelCase	,			sep_token=__UpperCamelCase	,			pad_token=__UpperCamelCase	,			**__UpperCamelCase	,			)

						try:
											import sacremoses
						except ImportError:
											raise ImportError(
											    'You need to install sacremoses to use XLMTokenizer. '
											    'See https://pypi.org/project/sacremoses/ for installation.'   )

						__magic_name__				=     sacremoses

						__magic_name__				=     src_vocab_file
						__magic_name__				=     tgt_vocab_file
						__magic_name__				=     merges_file
						__magic_name__				=     do_lower_case

						# cache of sm.MosesPunctNormalizer instance
						__magic_name__				=     {}
						# cache of sm.MosesTokenizer instance
						__magic_name__				=     {}
						__magic_name__				=     {}

						if langs and len(__UpperCamelCase   ) == 2:
											__magic_name__,						__magic_name__				=     langs
						else:
											raise ValueError(
											    F'arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '
											    'Usually that means that tokenizer can\'t find a mapping for the given model path '
											    'in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer.'   )

						with open(__UpperCamelCase	,			encoding='utf-8'   ) as src_vocab_handle:
											__magic_name__				=     json.load(__UpperCamelCase   )
						with open(__UpperCamelCase	,			encoding='utf-8'   ) as tgt_vocab_handle:
											__magic_name__				=     json.load(__UpperCamelCase   )
											__magic_name__				=     {v: k for k, v in tgt_vocab.items()}
						with open(__UpperCamelCase	,			encoding='utf-8'   ) as merges_handle:
											__magic_name__				=     merges_handle.read().split('\n'   )[:-1]
						__magic_name__				=     [tuple(merge.split()[:2]   ) for merge in merges]
						__magic_name__				=     dict(zip(__UpperCamelCase	,			range(len(__UpperCamelCase   )   )   )   )
						__magic_name__				=     {}







	def 						_SCREAMING_SNAKE_CASE				( self:			Any   ):

						'''simple docstring'''





						return self.get_src_vocab()







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			Optional[int]   ):

						'''simple docstring'''





						return self.src_vocab_size







	def 						_SCREAMING_SNAKE_CASE				( self:			Dict	,			__UpperCamelCase:			Dict	,			__UpperCamelCase:			Any   ):

						'''simple docstring'''





						if lang not in self.cache_moses_punct_normalizer:
											__magic_name__				=     self.sm.MosesPunctNormalizer(lang=__UpperCamelCase   )
											__magic_name__				=     punct_normalizer
						return self.cache_moses_punct_normalizer[lang].normalize(__UpperCamelCase   )







	def 						_SCREAMING_SNAKE_CASE				( self:			Optional[Any]	,			__UpperCamelCase:			str	,			__UpperCamelCase:			Dict   ):

						'''simple docstring'''





						if lang not in self.cache_moses_tokenizer:
											__magic_name__				=     self.sm.MosesTokenizer(lang=__UpperCamelCase   )
											__magic_name__				=     moses_tokenizer
						return self.cache_moses_tokenizer[lang].tokenize(
						    __UpperCamelCase	,			aggressive_dash_splits=__UpperCamelCase	,			return_str=__UpperCamelCase	,			escape=__UpperCamelCase   )







	def 						_SCREAMING_SNAKE_CASE				( self:			Optional[Any]	,			__UpperCamelCase:			Optional[Any]	,			__UpperCamelCase:			Optional[int]   ):

						'''simple docstring'''





						if lang not in self.cache_moses_detokenizer:
											__magic_name__				=     self.sm.MosesDetokenizer(lang=__UpperCamelCase   )
											__magic_name__				=     moses_detokenizer
						return self.cache_moses_detokenizer[lang].detokenize(__UpperCamelCase   )







	def 						_SCREAMING_SNAKE_CASE				( self:			Optional[int]	,			__UpperCamelCase:			int	,			__UpperCamelCase:			Union[str, Any]   ):

						'''simple docstring'''





						__magic_name__				=     replace_unicode_punct(__UpperCamelCase   )
						__magic_name__				=     self.moses_punct_norm(__UpperCamelCase	,			__UpperCamelCase   )
						__magic_name__				=     remove_non_printing_char(__UpperCamelCase   )
						return text







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			Union[str, Any]   ):

						'''simple docstring'''





						return len(self.encoder   )







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			List[Any]   ):

						'''simple docstring'''





						return len(self.decoder   )







	def 						_SCREAMING_SNAKE_CASE				( self:			Optional[int]   ):

						'''simple docstring'''





						return dict(self.encoder	,			**self.added_tokens_encoder   )







	def 						_SCREAMING_SNAKE_CASE				( self:			List[Any]   ):

						'''simple docstring'''





						return dict(self.decoder	,			**self.added_tokens_decoder   )







	def 						_SCREAMING_SNAKE_CASE				( self:			List[Any]	,			__UpperCamelCase:			Any   ):

						'''simple docstring'''





						__magic_name__				=     tuple(token[:-1]   ) + (token[-1] + '</w>',)
						if token in self.cache:
											return self.cache[token]
						__magic_name__				=     get_pairs(__UpperCamelCase   )

						if not pairs:
											return token + "</w>"

						while True:
											__magic_name__				=     min(__UpperCamelCase	,			key=lambda __UpperCamelCase   : self.bpe_ranks.get(__UpperCamelCase	,			float('inf'   )   )   )
											if bigram not in self.bpe_ranks:
																break
											__magic_name__,						__magic_name__				=     bigram
											__magic_name__				=     []
											__magic_name__				=     0
											while i < len(__UpperCamelCase   ):
																try:
																					__magic_name__				=     word.index(__UpperCamelCase	,			__UpperCamelCase   )
																except ValueError:
																					new_word.extend(word[i:]   )
																					break
																else:
																					new_word.extend(word[i:j]   )
																					__magic_name__				=     j

																if word[i] == first and i < len(__UpperCamelCase   ) - 1 and word[i + 1] == second:
																					new_word.append(first + second   )
																					i += 2
																else:
																					new_word.append(word[i]   )
																					i += 1
											__magic_name__				=     tuple(__UpperCamelCase   )
											__magic_name__				=     new_word
											if len(__UpperCamelCase   ) == 1:
																break
											else:
																__magic_name__				=     get_pairs(__UpperCamelCase   )
						__magic_name__				=     ' '.join(__UpperCamelCase   )
						if word == "\n  </w>":
											__magic_name__				=     '\n</w>'
						__magic_name__				=     word
						return word







	def 						_SCREAMING_SNAKE_CASE				( self:			List[Any]	,			__UpperCamelCase:			Tuple	,			__UpperCamelCase:			List[Any]="en"	,			__UpperCamelCase:			Optional[int]=False   ):

						'''simple docstring'''





						__magic_name__				=     self.src_lang

						if self.do_lower_case:
											__magic_name__				=     text.lower()

						if bypass_tokenizer:
											__magic_name__				=     text.split()
						else:
											__magic_name__				=     self.moses_pipeline(__UpperCamelCase	,			lang=__UpperCamelCase   )
											__magic_name__				=     self.moses_tokenize(__UpperCamelCase	,			lang=__UpperCamelCase   )

						__magic_name__				=     []
						for token in text:
											if token:
																split_tokens.extend(list(self.bpe(__UpperCamelCase   ).split(' '   )   )   )

						return split_tokens







	def 						_SCREAMING_SNAKE_CASE				( self:			List[str]	,			__UpperCamelCase:			List[str]   ):

						'''simple docstring'''





						return self.encoder.get(__UpperCamelCase	,			self.encoder.get(self.unk_token   )   )







	def 						_SCREAMING_SNAKE_CASE				( self:			int	,			__UpperCamelCase:			Any   ):

						'''simple docstring'''





						return self.decoder.get(__UpperCamelCase	,			self.unk_token   )







	def 						_SCREAMING_SNAKE_CASE				( self:			int	,			__UpperCamelCase:			Union[str, Any]   ):

						'''simple docstring'''





						__magic_name__				=     [t.replace(' '	,			''   ).replace('</w>'	,			' '   ) for t in tokens]
						__magic_name__				=     ''.join(__UpperCamelCase   ).split()
						# detokenize
						__magic_name__				=     self.moses_detokenize(__UpperCamelCase	,			self.tgt_lang   )
						return text







	def 						_SCREAMING_SNAKE_CASE				( self:			List[str]	,			__UpperCamelCase:			List[int]	,			__UpperCamelCase:			Optional[List[int]] = None   ):

						'''simple docstring'''





						__magic_name__				=     [self.sep_token_id]

						# no bos used in fairseq
						if token_ids_a is None:
											return token_ids_a + sep
						return token_ids_a + sep + token_ids_a + sep







	def 						_SCREAMING_SNAKE_CASE				( self:			Tuple	,			__UpperCamelCase:			List[int]	,			__UpperCamelCase:			Optional[List[int]] = None	,			__UpperCamelCase:			bool = False   ):

						'''simple docstring'''





						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=__UpperCamelCase	,			token_ids_a=__UpperCamelCase	,			already_has_special_tokens=__UpperCamelCase   )
						# no bos used in fairseq
						if token_ids_a is not None:
											return ([0] * len(__UpperCamelCase   )) + [1] + ([0] * len(__UpperCamelCase   )) + [1]
						return ([0] * len(__UpperCamelCase   )) + [1]







	def 						_SCREAMING_SNAKE_CASE				( self:			int	,			__UpperCamelCase:			List[int]	,			__UpperCamelCase:			Optional[List[int]] = None   ):

						'''simple docstring'''





						__magic_name__				=     [self.sep_token_id]

						# no bos used in fairseq
						if token_ids_a is None:
											return len(token_ids_a + sep   ) * [0]
						return len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







	def 						_SCREAMING_SNAKE_CASE				( self:			Union[str, Any]	,			__UpperCamelCase:			str	,			__UpperCamelCase:			Optional[str] = None   ):

						'''simple docstring'''





						if not os.path.isdir(__UpperCamelCase   ):
											logger.error(F'Vocabulary path ({save_directory}) should be a directory'   )
											return

						__magic_name__				=     os.path.join(
						    __UpperCamelCase	,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['src_vocab_file']   )
						__magic_name__				=     os.path.join(
						    __UpperCamelCase	,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['tgt_vocab_file']   )
						__magic_name__				=     os.path.join(
						    __UpperCamelCase	,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']   )

						with open(__UpperCamelCase	,			'w'	,			encoding='utf-8'   ) as f:
											f.write(json.dumps(self.encoder	,			indent=2	,			sort_keys=__UpperCamelCase	,			ensure_ascii=__UpperCamelCase   ) + '\n'   )

						with open(__UpperCamelCase	,			'w'	,			encoding='utf-8'   ) as f:
											__magic_name__				=     {v: k for k, v in self.decoder.items()}
											f.write(json.dumps(__UpperCamelCase	,			indent=2	,			sort_keys=__UpperCamelCase	,			ensure_ascii=__UpperCamelCase   ) + '\n'   )

						__magic_name__				=     0
						with open(__UpperCamelCase	,			'w'	,			encoding='utf-8'   ) as writer:
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,			key=lambda __UpperCamelCase   : kv[1]   ):
																if index != token_index:
																					logger.warning(
																					    F'Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'
																					    ' Please check that the tokenizer is not corrupted!'   )
																					__magic_name__				=     token_index
																writer.write(' '.join(__UpperCamelCase   ) + '\n'   )
																index += 1

						return src_vocab_file, tgt_vocab_file, merges_file







	def __getstate__( self:			Optional[int]   ):

						'''simple docstring'''





						__magic_name__				=     self.__dict__.copy()
						__magic_name__				=     None
						return state







	def __setstate__( self:			Optional[Any]	,			__UpperCamelCase:			Tuple   ):

						'''simple docstring'''





						__magic_name__				=     d

						try:
											import sacremoses
						except ImportError:
											raise ImportError(
											    'You need to install sacremoses to use XLMTokenizer. '
											    'See https://pypi.org/project/sacremoses/ for installation.'   )

						__magic_name__				=     sacremoses







from typing import Callable, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A__							=				logging.get_logger(__name__)

A__							=				{
    "microsoft/prophetnet-large-uncased": (
        "https://huggingface.co/microsoft/prophetnet-large-uncased/resolve/main/config.json"
    ),
}

class 					__UpperCamelCase					(       SCREAMING_SNAKE_CASE      ):
	_lowercase	: Union[str, Any]												=	"prophetnet"
	_lowercase	: Any												=	["past_key_values"]
	_lowercase	: int												=	{
	    "num_attention_heads": "num_encoder_attention_heads",
	}







	def __init__( self:			Union[str, Any]	,			__UpperCamelCase:			Optional[float] = 0.1	,			__UpperCamelCase:			Optional[Union[str, Callable]] = "gelu"	,			__UpperCamelCase:			Optional[int] = 3_05_22	,			__UpperCamelCase:			Optional[int] = 10_24	,			__UpperCamelCase:			Optional[int] = 40_96	,			__UpperCamelCase:			Optional[int] = 12	,			__UpperCamelCase:			Optional[int] = 16	,			__UpperCamelCase:			Optional[int] = 40_96	,			__UpperCamelCase:			Optional[int] = 12	,			__UpperCamelCase:			Optional[int] = 16	,			__UpperCamelCase:			Optional[float] = 0.1	,			__UpperCamelCase:			Optional[float] = 0.1	,			__UpperCamelCase:			Optional[int] = 5_12	,			__UpperCamelCase:			Optional[float] = 0.02	,			__UpperCamelCase:			Optional[bool] = True	,			__UpperCamelCase:			Optional[bool] = True	,			__UpperCamelCase:			Optional[int] = 0	,			__UpperCamelCase:			Optional[int] = 2	,			__UpperCamelCase:			Optional[int] = 32	,			__UpperCamelCase:			Optional[int] = 1_28	,			__UpperCamelCase:			Optional[bool] = False	,			__UpperCamelCase:			Optional[float] = 0.0	,			__UpperCamelCase:			Optional[bool] = True	,			__UpperCamelCase:			Optional[int] = 0	,			__UpperCamelCase:			Optional[int] = 1	,			__UpperCamelCase:			Optional[int] = 2	,			**__UpperCamelCase:			Optional[int]	,			):

						'''simple docstring'''





						__magic_name__				=     vocab_size
						__magic_name__				=     hidden_size
						__magic_name__				=     encoder_ffn_dim
						__magic_name__				=     num_encoder_layers
						__magic_name__				=     num_encoder_attention_heads
						__magic_name__				=     decoder_ffn_dim
						__magic_name__				=     num_decoder_layers
						__magic_name__				=     num_decoder_attention_heads
						__magic_name__				=     max_position_embeddings
						__magic_name__				=     init_std  # Normal(0, this parameter)
						__magic_name__				=     activation_function

						# parameters for prophetnet
						__magic_name__				=     ngram
						__magic_name__				=     num_buckets
						__magic_name__				=     relative_max_distance
						__magic_name__				=     disable_ngram_loss
						__magic_name__				=     eps

						# 3 Types of Dropout
						__magic_name__				=     attention_dropout
						__magic_name__				=     activation_dropout
						__magic_name__				=     dropout

						__magic_name__				=     use_cache

						super().__init__(
						    pad_token_id=__UpperCamelCase	,			bos_token_id=__UpperCamelCase	,			eos_token_id=__UpperCamelCase	,			is_encoder_decoder=__UpperCamelCase	,			add_cross_attention=__UpperCamelCase	,			decoder_start_token_id=__UpperCamelCase	,			**__UpperCamelCase	,			)







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			Tuple   ):

						'''simple docstring'''





						return self.num_encoder_layers + self.num_decoder_layers







	@num_hidden_layers.setter
	def 						_SCREAMING_SNAKE_CASE				( self:			Union[str, Any]	,			__UpperCamelCase:			List[Any]   ):

						'''simple docstring'''





						raise NotImplementedError(
						    'This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and'
						    ' `num_decoder_layers`.'   )




