


"""simple docstring"""






import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__							:Union[str, Any]       =  logging.get_logger(__name__)

UpperCamelCase__							:List[Any]       =  {
    """src_vocab_file""": """vocab-src.json""",
    """tgt_vocab_file""": """vocab-tgt.json""",
    """merges_file""": """merges.txt""",
}

UpperCamelCase__							:Tuple       =  {
    """src_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"""
    },
    """tgt_vocab_file""": {
        """stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"""
    },
    """merges_file""": {"""stas/tiny-wmt19-en-de""": """https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"""},
}

UpperCamelCase__							:str       =  {"""stas/tiny-wmt19-en-de""": 1_024}
UpperCamelCase__							:str       =  {
    """stas/tiny-wmt19-en-de""": {
        """langs""": ["""en""", """de"""],
        """model_max_length""": 1_024,
        """special_tokens_map_file""": None,
        """full_tokenizer_file""": None,
    }
}





def 							A_					(   snake_case__		)				->     Tuple:
			_UpperCamelCase				:Tuple				    =  set()
			_UpperCamelCase				:int				    =  word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)		)
						_UpperCamelCase				:Union[str, Any]				    =  char
			return pairs





def 							A_					(   snake_case__		)				->     Union[str, Any]:
			_UpperCamelCase				:List[Any]				    =  text.replace('''，'''					,				''','''		)
			_UpperCamelCase				:List[Any]				    =  re.sub(R'''。\s*'''					,				'''. '''					,				snake_case__		)
			_UpperCamelCase				:Any				    =  text.replace('''、'''					,				''','''		)
			_UpperCamelCase				:str				    =  text.replace('''”'''					,				'''"'''		)
			_UpperCamelCase				:int				    =  text.replace('''“'''					,				'''"'''		)
			_UpperCamelCase				:int				    =  text.replace('''∶'''					,				''':'''		)
			_UpperCamelCase				:List[str]				    =  text.replace('''：'''					,				''':'''		)
			_UpperCamelCase				:Optional[Any]				    =  text.replace('''？'''					,				'''?'''		)
			_UpperCamelCase				:str				    =  text.replace('''《'''					,				'''"'''		)
			_UpperCamelCase				:str				    =  text.replace('''》'''					,				'''"'''		)
			_UpperCamelCase				:Optional[Any]				    =  text.replace('''）'''					,				''')'''		)
			_UpperCamelCase				:Any				    =  text.replace('''！'''					,				'''!'''		)
			_UpperCamelCase				:int				    =  text.replace('''（'''					,				'''('''		)
			_UpperCamelCase				:str				    =  text.replace('''；'''					,				''';'''		)
			_UpperCamelCase				:List[str]				    =  text.replace('''１'''					,				'''1'''		)
			_UpperCamelCase				:int				    =  text.replace('''」'''					,				'''"'''		)
			_UpperCamelCase				:List[Any]				    =  text.replace('''「'''					,				'''"'''		)
			_UpperCamelCase				:Optional[Any]				    =  text.replace('''０'''					,				'''0'''		)
			_UpperCamelCase				:Dict				    =  text.replace('''３'''					,				'''3'''		)
			_UpperCamelCase				:Optional[Any]				    =  text.replace('''２'''					,				'''2'''		)
			_UpperCamelCase				:Dict				    =  text.replace('''５'''					,				'''5'''		)
			_UpperCamelCase				:List[str]				    =  text.replace('''６'''					,				'''6'''		)
			_UpperCamelCase				:Tuple				    =  text.replace('''９'''					,				'''9'''		)
			_UpperCamelCase				:List[Any]				    =  text.replace('''７'''					,				'''7'''		)
			_UpperCamelCase				:Optional[int]				    =  text.replace('''８'''					,				'''8'''		)
			_UpperCamelCase				:Any				    =  text.replace('''４'''					,				'''4'''		)
			_UpperCamelCase				:str				    =  re.sub(R'''．\s*'''					,				'''. '''					,				snake_case__		)
			_UpperCamelCase				:Dict				    =  text.replace('''～'''					,				'''~'''		)
			_UpperCamelCase				:int				    =  text.replace('''’'''					,				'''\''''		)
			_UpperCamelCase				:Union[str, Any]				    =  text.replace('''…'''					,				'''...'''		)
			_UpperCamelCase				:Union[str, Any]				    =  text.replace('''━'''					,				'''-'''		)
			_UpperCamelCase				:Dict				    =  text.replace('''〈'''					,				'''<'''		)
			_UpperCamelCase				:Union[str, Any]				    =  text.replace('''〉'''					,				'''>'''		)
			_UpperCamelCase				:Optional[int]				    =  text.replace('''【'''					,				'''['''		)
			_UpperCamelCase				:List[Any]				    =  text.replace('''】'''					,				''']'''		)
			_UpperCamelCase				:List[Any]				    =  text.replace('''％'''					,				'''%'''		)
			return text





def 							A_					(   snake_case__		)				->     Tuple:
			_UpperCamelCase				:Any				    =  []
			for char in text:
						_UpperCamelCase				:int				    =  unicodedata.category(snake_case__		)
						if cat.startswith('''C'''		):
									continue
						output.append(snake_case__		)
			return "".join(snake_case__		)






class     A(     lowerCamelCase__		):

		"""simple docstring"""





		A         =       VOCAB_FILES_NAMES
		A         =       PRETRAINED_VOCAB_FILES_MAP
		A         =       PRETRAINED_INIT_CONFIGURATION
		A         =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		A         =       ["input_ids", "attention_mask"]
		def __init__(   self				,							SCREAMING_SNAKE_CASE__=None				,							SCREAMING_SNAKE_CASE__=None				,							SCREAMING_SNAKE_CASE__=None				,							SCREAMING_SNAKE_CASE__=None				,							SCREAMING_SNAKE_CASE__=False				,							SCREAMING_SNAKE_CASE__="<unk>"				,							SCREAMING_SNAKE_CASE__="<s>"				,							SCREAMING_SNAKE_CASE__="</s>"				,							SCREAMING_SNAKE_CASE__="<pad>"				,							**SCREAMING_SNAKE_CASE__				,							)			->  List[Any]:

					"""simple docstring"""



					super().__init__(
					    langs=SCREAMING_SNAKE_CASE__				,							src_vocab_file=SCREAMING_SNAKE_CASE__				,							tgt_vocab_file=SCREAMING_SNAKE_CASE__				,							merges_file=SCREAMING_SNAKE_CASE__				,							do_lower_case=SCREAMING_SNAKE_CASE__				,							unk_token=SCREAMING_SNAKE_CASE__				,							bos_token=SCREAMING_SNAKE_CASE__				,							sep_token=SCREAMING_SNAKE_CASE__				,							pad_token=SCREAMING_SNAKE_CASE__				,							**SCREAMING_SNAKE_CASE__				,							)

					try:
								import sacremoses
					except ImportError:
								raise ImportError(
								    '''You need to install sacremoses to use XLMTokenizer. '''
								    '''See https://pypi.org/project/sacremoses/ for installation.'''      )

					_UpperCamelCase				:Dict				    =  sacremoses

					_UpperCamelCase				:List[Any]				    =  src_vocab_file
					_UpperCamelCase				:List[Any]				    =  tgt_vocab_file
					_UpperCamelCase				:Optional[Any]				    =  merges_file
					_UpperCamelCase				:List[Any]				    =  do_lower_case

					# cache of sm.MosesPunctNormalizer instance
					_UpperCamelCase				:Optional[int]				    =  {}
					# cache of sm.MosesTokenizer instance
					_UpperCamelCase				:str				    =  {}
					_UpperCamelCase				:Dict				    =  {}

					if langs and len(SCREAMING_SNAKE_CASE__      ) == 2:
								_UpperCamelCase    ,			_UpperCamelCase				:Union[str, Any]				    =  langs
					else:
								raise ValueError(
								    f"arg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got {langs}. "
								    '''Usually that means that tokenizer can\'t find a mapping for the given model path '''
								    '''in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer.'''      )

					with open(SCREAMING_SNAKE_CASE__				,							encoding='''utf-8'''      ) as src_vocab_handle:
								_UpperCamelCase				:Dict				    =  json.load(SCREAMING_SNAKE_CASE__      )
					with open(SCREAMING_SNAKE_CASE__				,							encoding='''utf-8'''      ) as tgt_vocab_handle:
								_UpperCamelCase				:Optional[Any]				    =  json.load(SCREAMING_SNAKE_CASE__      )
								_UpperCamelCase				:Dict				    =  {v: k for k, v in tgt_vocab.items()}
					with open(SCREAMING_SNAKE_CASE__				,							encoding='''utf-8'''      ) as merges_handle:
								_UpperCamelCase				:Any				    =  merges_handle.read().split('''\n'''      )[:-1]
					_UpperCamelCase				:Optional[Any]				    =  [tuple(merge.split()[:2]      ) for merge in merges]
					_UpperCamelCase				:Any				    =  dict(zip(SCREAMING_SNAKE_CASE__				,							range(len(SCREAMING_SNAKE_CASE__      )      )      )      )
					_UpperCamelCase				:Optional[int]				    =  {}
		def      _UpperCamelCase(   self      )			->  Dict[str, int]:

					"""simple docstring"""



					return self.get_src_vocab()
		@property
		def      _UpperCamelCase(   self      )			->  int:

					"""simple docstring"""



					return self.src_vocab_size
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__      )			->  str:

					"""simple docstring"""



					if lang not in self.cache_moses_punct_normalizer:
								_UpperCamelCase				:int				    =  self.sm.MosesPunctNormalizer(lang=SCREAMING_SNAKE_CASE__      )
								_UpperCamelCase				:Tuple				    =  punct_normalizer
					return self.cache_moses_punct_normalizer[lang].normalize(SCREAMING_SNAKE_CASE__      )
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__      )			->  Union[str, Any]:

					"""simple docstring"""



					if lang not in self.cache_moses_tokenizer:
								_UpperCamelCase				:Any				    =  self.sm.MosesTokenizer(lang=SCREAMING_SNAKE_CASE__      )
								_UpperCamelCase				:Tuple				    =  moses_tokenizer
					return self.cache_moses_tokenizer[lang].tokenize(
					    SCREAMING_SNAKE_CASE__				,							aggressive_dash_splits=SCREAMING_SNAKE_CASE__				,							return_str=SCREAMING_SNAKE_CASE__				,							escape=SCREAMING_SNAKE_CASE__      )
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__      )			->  Any:

					"""simple docstring"""



					if lang not in self.cache_moses_detokenizer:
								_UpperCamelCase				:str				    =  self.sm.MosesDetokenizer(lang=SCREAMING_SNAKE_CASE__      )
								_UpperCamelCase				:Dict				    =  moses_detokenizer
					return self.cache_moses_detokenizer[lang].detokenize(SCREAMING_SNAKE_CASE__      )
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__      )			->  Union[str, Any]:

					"""simple docstring"""



					_UpperCamelCase				:Union[str, Any]				    =  replace_unicode_punct(SCREAMING_SNAKE_CASE__      )
					_UpperCamelCase				:List[Any]				    =  self.moses_punct_norm(SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__      )
					_UpperCamelCase				:Dict				    =  remove_non_printing_char(SCREAMING_SNAKE_CASE__      )
					return text
		@property
		def      _UpperCamelCase(   self      )			->  Dict:

					"""simple docstring"""



					return len(self.encoder      )
		@property
		def      _UpperCamelCase(   self      )			->  List[Any]:

					"""simple docstring"""



					return len(self.decoder      )
		def      _UpperCamelCase(   self      )			->  int:

					"""simple docstring"""



					return dict(self.encoder				,							**self.added_tokens_encoder      )
		def      _UpperCamelCase(   self      )			->  str:

					"""simple docstring"""



					return dict(self.decoder				,							**self.added_tokens_decoder      )
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__      )			->  List[Any]:

					"""simple docstring"""



					_UpperCamelCase				:str				    =  tuple(token[:-1]      ) + (token[-1] + '''</w>''',)
					if token in self.cache:
								return self.cache[token]
					_UpperCamelCase				:Optional[Any]				    =  get_pairs(SCREAMING_SNAKE_CASE__      )

					if not pairs:
								return token + "</w>"

					while True:
								_UpperCamelCase				:str				    =  min(SCREAMING_SNAKE_CASE__				,							key=lambda SCREAMING_SNAKE_CASE__      : self.bpe_ranks.get(SCREAMING_SNAKE_CASE__				,							float('''inf'''      )      )      )
								if bigram not in self.bpe_ranks:
											break
								_UpperCamelCase    ,			_UpperCamelCase				:Optional[Any]				    =  bigram
								_UpperCamelCase				:Any				    =  []
								_UpperCamelCase				:List[Any]				    =  0
								while i < len(SCREAMING_SNAKE_CASE__      ):
											try:
														_UpperCamelCase				:Union[str, Any]				    =  word.index(SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__      )
											except ValueError:
														new_word.extend(word[i:]      )
														break
											else:
														new_word.extend(word[i:j]      )
														_UpperCamelCase				:int				    =  j

											if word[i] == first and i < len(SCREAMING_SNAKE_CASE__      ) - 1 and word[i + 1] == second:
														new_word.append(first + second      )
														i += 2
											else:
														new_word.append(word[i]      )
														i += 1
								_UpperCamelCase				:List[Any]				    =  tuple(SCREAMING_SNAKE_CASE__      )
								_UpperCamelCase				:int				    =  new_word
								if len(SCREAMING_SNAKE_CASE__      ) == 1:
											break
								else:
											_UpperCamelCase				:int				    =  get_pairs(SCREAMING_SNAKE_CASE__      )
					_UpperCamelCase				:List[Any]				    =  ''' '''.join(SCREAMING_SNAKE_CASE__      )
					if word == "\n  </w>":
								_UpperCamelCase				:List[Any]				    =  '''\n</w>'''
					_UpperCamelCase				:Any				    =  word
					return word
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__="en"				,							SCREAMING_SNAKE_CASE__=False      )			->  str:

					"""simple docstring"""



					_UpperCamelCase				:Optional[int]				    =  self.src_lang

					if self.do_lower_case:
								_UpperCamelCase				:Union[str, Any]				    =  text.lower()

					if bypass_tokenizer:
								_UpperCamelCase				:Tuple				    =  text.split()
					else:
								_UpperCamelCase				:Union[str, Any]				    =  self.moses_pipeline(SCREAMING_SNAKE_CASE__				,							lang=SCREAMING_SNAKE_CASE__      )
								_UpperCamelCase				:Union[str, Any]				    =  self.moses_tokenize(SCREAMING_SNAKE_CASE__				,							lang=SCREAMING_SNAKE_CASE__      )

					_UpperCamelCase				:Tuple				    =  []
					for token in text:
								if token:
											split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE__      ).split(''' '''      )      )      )

					return split_tokens
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__      )			->  Any:

					"""simple docstring"""



					return self.encoder.get(SCREAMING_SNAKE_CASE__				,							self.encoder.get(self.unk_token      )      )
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__      )			->  Union[str, Any]:

					"""simple docstring"""



					return self.decoder.get(SCREAMING_SNAKE_CASE__				,							self.unk_token      )
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__      )			->  str:

					"""simple docstring"""



					_UpperCamelCase				:Optional[Any]				    =  [t.replace(''' '''				,							''''''      ).replace('''</w>'''				,							''' '''      ) for t in tokens]
					_UpperCamelCase				:List[Any]				    =  ''''''.join(SCREAMING_SNAKE_CASE__      ).split()
					# detokenize
					_UpperCamelCase				:Optional[int]				    =  self.moses_detokenize(SCREAMING_SNAKE_CASE__				,							self.tgt_lang      )
					return text
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__ = None      )			->  List[int]:

					"""simple docstring"""



					_UpperCamelCase				:int				    =  [self.sep_token_id]

					# no bos used in fairseq
					if token_ids_a is None:
								return token_ids_a + sep
					return token_ids_a + sep + token_ids_a + sep
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__ = None				,							SCREAMING_SNAKE_CASE__ = False      )			->  List[int]:

					"""simple docstring"""



					if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=SCREAMING_SNAKE_CASE__				,							token_ids_a=SCREAMING_SNAKE_CASE__				,							already_has_special_tokens=SCREAMING_SNAKE_CASE__      )
					# no bos used in fairseq
					if token_ids_a is not None:
								return ([0] * len(SCREAMING_SNAKE_CASE__      )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE__      )) + [1]
					return ([0] * len(SCREAMING_SNAKE_CASE__      )) + [1]
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__ = None      )			->  List[int]:

					"""simple docstring"""



					_UpperCamelCase				:str				    =  [self.sep_token_id]

					# no bos used in fairseq
					if token_ids_a is None:
								return len(token_ids_a + sep      ) * [0]
					return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]
		def      _UpperCamelCase(   self				,							SCREAMING_SNAKE_CASE__				,							SCREAMING_SNAKE_CASE__ = None      )			->  Tuple[str]:

					"""simple docstring"""



					if not os.path.isdir(SCREAMING_SNAKE_CASE__      ):
								logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
								return

					_UpperCamelCase				:Any				    =  os.path.join(
					    SCREAMING_SNAKE_CASE__				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''src_vocab_file''']      )
					_UpperCamelCase				:Union[str, Any]				    =  os.path.join(
					    SCREAMING_SNAKE_CASE__				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''tgt_vocab_file''']      )
					_UpperCamelCase				:Tuple				    =  os.path.join(
					    SCREAMING_SNAKE_CASE__				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']      )

					with open(SCREAMING_SNAKE_CASE__				,							'''w'''				,							encoding='''utf-8'''      ) as f:
								f.write(json.dumps(self.encoder				,							indent=2				,							sort_keys=SCREAMING_SNAKE_CASE__				,							ensure_ascii=SCREAMING_SNAKE_CASE__      ) + '''\n'''      )

					with open(SCREAMING_SNAKE_CASE__				,							'''w'''				,							encoding='''utf-8'''      ) as f:
								_UpperCamelCase				:Union[str, Any]				    =  {v: k for k, v in self.decoder.items()}
								f.write(json.dumps(SCREAMING_SNAKE_CASE__				,							indent=2				,							sort_keys=SCREAMING_SNAKE_CASE__				,							ensure_ascii=SCREAMING_SNAKE_CASE__      ) + '''\n'''      )

					_UpperCamelCase				:Any				    =  0
					with open(SCREAMING_SNAKE_CASE__				,							'''w'''				,							encoding='''utf-8'''      ) as writer:
								for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,							key=lambda SCREAMING_SNAKE_CASE__      : kv[1]      ):
											if index != token_index:
														logger.warning(
														    f"Saving vocabulary to {merges_file}: BPE merge indices are not consecutive."
														    ''' Please check that the tokenizer is not corrupted!'''      )
														_UpperCamelCase				:Optional[int]				    =  token_index
											writer.write(''' '''.join(SCREAMING_SNAKE_CASE__      ) + '''\n'''      )
											index += 1

					return src_vocab_file, tgt_vocab_file, merges_file
		def __getstate__(   self      )			->  str:

					"""simple docstring"""



					_UpperCamelCase				:List[Any]				    =  self.__dict__.copy()
					_UpperCamelCase				:List[str]				    =  None
					return state





		def __setstate__(   self				,							SCREAMING_SNAKE_CASE__      )			->  int:

					"""simple docstring"""



					_UpperCamelCase				:Union[str, Any]				    =  d

					try:
								import sacremoses
					except ImportError:
								raise ImportError(
								    '''You need to install sacremoses to use XLMTokenizer. '''
								    '''See https://pypi.org/project/sacremoses/ for installation.'''      )

					_UpperCamelCase				:Optional[Any]				    =  sacremoses





"""simple docstring"""






from dataclasses import dataclass
from typing import List, Optional, Union

import numpy as np
import PIL

from ...utils import BaseOutput, is_invisible_watermark_available, is_torch_available, is_transformers_available






@dataclass
class     A(     lowerCamelCase__		):

		"""simple docstring"""





		A         =       42


if is_transformers_available() and is_torch_available() and is_invisible_watermark_available():
					from .pipeline_stable_diffusion_xl import StableDiffusionXLPipeline
					from .pipeline_stable_diffusion_xl_imgaimg import StableDiffusionXLImgaImgPipeline

