

import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase     :       Tuple        =       logging.get_logger(__name__)

lowercase     :       Dict        =       {
    "src_vocab_file": "vocab-src.json",
    "tgt_vocab_file": "vocab-tgt.json",
    "merges_file": "merges.txt",
}

lowercase     :       List[Any]        =       {
    "src_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json"
    },
    "tgt_vocab_file": {
        "stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json"
    },
    "merges_file": {"stas/tiny-wmt19-en-de": "https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt"},
}

lowercase     :       Union[str, Any]        =       {"stas/tiny-wmt19-en-de": 1024}
lowercase     :       Optional[int]        =       {
    "stas/tiny-wmt19-en-de": {
        "langs": ["en", "de"],
        "model_max_length": 1024,
        "special_tokens_map_file": None,
        "full_tokenizer_file": None,
    }
}







def  UpperCAmelCase_   (_lowerCAmelCase :		Any		):
			__UpperCamelCase    :    Optional[Any]								=      set()
			__UpperCamelCase    :    int								=      word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)		)
						__UpperCamelCase    :    Dict								=      char
			return pairs







def  UpperCAmelCase_   (_lowerCAmelCase :		Any		):
			__UpperCamelCase    :    Optional[Any]								=      text.replace("，"     ,			","		)
			__UpperCamelCase    :    str								=      re.sub(R"。\s*"     ,			". "     ,			_lowerCAmelCase		)
			__UpperCamelCase    :    Optional[Any]								=      text.replace("、"     ,			","		)
			__UpperCamelCase    :    Any								=      text.replace("”"     ,			"\""		)
			__UpperCamelCase    :    Union[str, Any]								=      text.replace("“"     ,			"\""		)
			__UpperCamelCase    :    Optional[Any]								=      text.replace("∶"     ,			":"		)
			__UpperCamelCase    :    Union[str, Any]								=      text.replace("："     ,			":"		)
			__UpperCamelCase    :    Dict								=      text.replace("？"     ,			"?"		)
			__UpperCamelCase    :    Any								=      text.replace("《"     ,			"\""		)
			__UpperCamelCase    :    Tuple								=      text.replace("》"     ,			"\""		)
			__UpperCamelCase    :    List[str]								=      text.replace("）"     ,			")"		)
			__UpperCamelCase    :    Dict								=      text.replace("！"     ,			"!"		)
			__UpperCamelCase    :    Tuple								=      text.replace("（"     ,			"("		)
			__UpperCamelCase    :    Dict								=      text.replace("；"     ,			";"		)
			__UpperCamelCase    :    Optional[int]								=      text.replace("１"     ,			"1"		)
			__UpperCamelCase    :    Any								=      text.replace("」"     ,			"\""		)
			__UpperCamelCase    :    str								=      text.replace("「"     ,			"\""		)
			__UpperCamelCase    :    Tuple								=      text.replace("０"     ,			"0"		)
			__UpperCamelCase    :    Tuple								=      text.replace("３"     ,			"3"		)
			__UpperCamelCase    :    Optional[Any]								=      text.replace("２"     ,			"2"		)
			__UpperCamelCase    :    Any								=      text.replace("５"     ,			"5"		)
			__UpperCamelCase    :    Tuple								=      text.replace("６"     ,			"6"		)
			__UpperCamelCase    :    Any								=      text.replace("９"     ,			"9"		)
			__UpperCamelCase    :    Union[str, Any]								=      text.replace("７"     ,			"7"		)
			__UpperCamelCase    :    str								=      text.replace("８"     ,			"8"		)
			__UpperCamelCase    :    Optional[Any]								=      text.replace("４"     ,			"4"		)
			__UpperCamelCase    :    Dict								=      re.sub(R"．\s*"     ,			". "     ,			_lowerCAmelCase		)
			__UpperCamelCase    :    int								=      text.replace("～"     ,			"~"		)
			__UpperCamelCase    :    Optional[int]								=      text.replace("’"     ,			"'"		)
			__UpperCamelCase    :    Any								=      text.replace("…"     ,			"..."		)
			__UpperCamelCase    :    str								=      text.replace("━"     ,			"-"		)
			__UpperCamelCase    :    str								=      text.replace("〈"     ,			"<"		)
			__UpperCamelCase    :    Tuple								=      text.replace("〉"     ,			">"		)
			__UpperCamelCase    :    Dict								=      text.replace("【"     ,			"["		)
			__UpperCamelCase    :    int								=      text.replace("】"     ,			"]"		)
			__UpperCamelCase    :    Optional[Any]								=      text.replace("％"     ,			"%"		)
			return text







def  UpperCAmelCase_   (_lowerCAmelCase :		Tuple		):
			__UpperCamelCase    :    Dict								=      []
			for char in text:
						__UpperCamelCase    :    Union[str, Any]								=      unicodedata.category(_lowerCAmelCase		)
						if cat.startswith("C"		):
									continue
						output.append(_lowerCAmelCase		)
			return "".join(_lowerCAmelCase		)

class 		SCREAMING_SNAKE_CASE__					(		lowerCamelCase__   ):







						"""simple docstring"""




						lowercase  :				Optional[Any]											=							VOCAB_FILES_NAMES
						lowercase  :				Optional[Any]											=							PRETRAINED_VOCAB_FILES_MAP
						lowercase  :				Any											=							PRETRAINED_INIT_CONFIGURATION
						lowercase  :				Tuple											=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowercase  :				Tuple											=							['input_ids', 'attention_mask']

						def __init__(				self , __UpperCamelCase=None , __UpperCamelCase=None , __UpperCamelCase=None , __UpperCamelCase=None , __UpperCamelCase=False , __UpperCamelCase="<unk>" , __UpperCamelCase="<s>" , __UpperCamelCase="</s>" , __UpperCamelCase="<pad>" , **__UpperCamelCase , )							->	Optional[int]:




									'''simple docstring'''


									super().__init__(
									    langs=__UpperCamelCase , src_vocab_file=__UpperCamelCase , tgt_vocab_file=__UpperCamelCase , merges_file=__UpperCamelCase , do_lower_case=__UpperCamelCase , unk_token=__UpperCamelCase , bos_token=__UpperCamelCase , sep_token=__UpperCamelCase , pad_token=__UpperCamelCase , **__UpperCamelCase , )

									try:
												import sacremoses
									except ImportError:
												raise ImportError(
												    "You need to install sacremoses to use XLMTokenizer. "
												    "See https://pypi.org/project/sacremoses/ for installation."    )

									__UpperCamelCase    :    int								=      sacremoses

									__UpperCamelCase    :    Optional[Any]								=      src_vocab_file
									__UpperCamelCase    :    Optional[int]								=      tgt_vocab_file
									__UpperCamelCase    :    Union[str, Any]								=      merges_file
									__UpperCamelCase    :    Optional[Any]								=      do_lower_case

									# cache of sm.MosesPunctNormalizer instance
									__UpperCamelCase    :    str								=      {}
									# cache of sm.MosesTokenizer instance
									__UpperCamelCase    :    str								=      {}
									__UpperCamelCase    :    Dict								=      {}

									if langs and len(__UpperCamelCase    ) == 2:
												__UpperCamelCase    :    int								=      langs
									else:
												raise ValueError(
												    f'''arg `langs` needs to be a list of 2 langs, e.g. [\'en\', \'ru\'], but got {langs}. '''
												    "Usually that means that tokenizer can't find a mapping for the given model path "
												    "in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer."    )

									with open(__UpperCamelCase , encoding="utf-8"    ) as src_vocab_handle:
												__UpperCamelCase    :    Union[str, Any]								=      json.load(__UpperCamelCase    )
									with open(__UpperCamelCase , encoding="utf-8"    ) as tgt_vocab_handle:
												__UpperCamelCase    :    Optional[int]								=      json.load(__UpperCamelCase    )
												__UpperCamelCase    :    str								=      {v: k for k, v in tgt_vocab.items()}
									with open(__UpperCamelCase , encoding="utf-8"    ) as merges_handle:
												__UpperCamelCase    :    str								=      merges_handle.read().split("\n"    )[:-1]
									__UpperCamelCase    :    List[Any]								=      [tuple(merge.split()[:2]    ) for merge in merges]
									__UpperCamelCase    :    str								=      dict(zip(__UpperCamelCase , range(len(__UpperCamelCase    )    )    )    )
									__UpperCamelCase    :    str								=      {}

						def __lowerCamelCase					(				self    )							->	Dict[str, int]:




									'''simple docstring'''


									return self.get_src_vocab()

						@property
						def __lowerCamelCase					(				self    )							->	int:




									'''simple docstring'''


									return self.src_vocab_size

						def __lowerCamelCase					(				self , __UpperCamelCase , __UpperCamelCase    )							->	str:




									'''simple docstring'''


									if lang not in self.cache_moses_punct_normalizer:
												__UpperCamelCase    :    int								=      self.sm.MosesPunctNormalizer(lang=__UpperCamelCase    )
												__UpperCamelCase    :    Optional[Any]								=      punct_normalizer
									return self.cache_moses_punct_normalizer[lang].normalize(__UpperCamelCase    )

						def __lowerCamelCase					(				self , __UpperCamelCase , __UpperCamelCase    )							->	Any:




									'''simple docstring'''


									if lang not in self.cache_moses_tokenizer:
												__UpperCamelCase    :    List[Any]								=      self.sm.MosesTokenizer(lang=__UpperCamelCase    )
												__UpperCamelCase    :    Optional[int]								=      moses_tokenizer
									return self.cache_moses_tokenizer[lang].tokenize(
									    __UpperCamelCase , aggressive_dash_splits=__UpperCamelCase , return_str=__UpperCamelCase , escape=__UpperCamelCase    )

						def __lowerCamelCase					(				self , __UpperCamelCase , __UpperCamelCase    )							->	Optional[int]:




									'''simple docstring'''


									if lang not in self.cache_moses_detokenizer:
												__UpperCamelCase    :    Tuple								=      self.sm.MosesDetokenizer(lang=__UpperCamelCase    )
												__UpperCamelCase    :    int								=      moses_detokenizer
									return self.cache_moses_detokenizer[lang].detokenize(__UpperCamelCase    )

						def __lowerCamelCase					(				self , __UpperCamelCase , __UpperCamelCase    )							->	Tuple:




									'''simple docstring'''


									__UpperCamelCase    :    List[Any]								=      replace_unicode_punct(__UpperCamelCase    )
									__UpperCamelCase    :    Tuple								=      self.moses_punct_norm(__UpperCamelCase , __UpperCamelCase    )
									__UpperCamelCase    :    Optional[Any]								=      remove_non_printing_char(__UpperCamelCase    )
									return text

						@property
						def __lowerCamelCase					(				self    )							->	int:




									'''simple docstring'''


									return len(self.encoder    )

						@property
						def __lowerCamelCase					(				self    )							->	str:




									'''simple docstring'''


									return len(self.decoder    )

						def __lowerCamelCase					(				self    )							->	int:




									'''simple docstring'''


									return dict(self.encoder , **self.added_tokens_encoder    )

						def __lowerCamelCase					(				self    )							->	Tuple:




									'''simple docstring'''


									return dict(self.decoder , **self.added_tokens_decoder    )

						def __lowerCamelCase					(				self , __UpperCamelCase    )							->	Optional[int]:




									'''simple docstring'''


									__UpperCamelCase    :    Tuple								=      tuple(token[:-1]    ) + (token[-1] + "</w>",)
									if token in self.cache:
												return self.cache[token]
									__UpperCamelCase    :    Dict								=      get_pairs(__UpperCamelCase    )

									if not pairs:
												return token + "</w>"

									while True:
												__UpperCamelCase    :    List[Any]								=      min(__UpperCamelCase , key=lambda __UpperCamelCase    : self.bpe_ranks.get(__UpperCamelCase , float("inf"    )    )    )
												if bigram not in self.bpe_ranks:
															break
												__UpperCamelCase    :    List[Any]								=      bigram
												__UpperCamelCase    :    List[Any]								=      []
												__UpperCamelCase    :    Union[str, Any]								=      0
												while i < len(__UpperCamelCase    ):
															try:
																		__UpperCamelCase    :    int								=      word.index(__UpperCamelCase , __UpperCamelCase    )
															except ValueError:
																		new_word.extend(word[i:]    )
																		break
															else:
																		new_word.extend(word[i:j]    )
																		__UpperCamelCase    :    int								=      j

															if word[i] == first and i < len(__UpperCamelCase    ) - 1 and word[i + 1] == second:
																		new_word.append(first + second    )
																		i += 2
															else:
																		new_word.append(word[i]    )
																		i += 1
												__UpperCamelCase    :    Dict								=      tuple(__UpperCamelCase    )
												__UpperCamelCase    :    Union[str, Any]								=      new_word
												if len(__UpperCamelCase    ) == 1:
															break
												else:
															__UpperCamelCase    :    str								=      get_pairs(__UpperCamelCase    )
									__UpperCamelCase    :    Any								=      " ".join(__UpperCamelCase    )
									if word == "\n  </w>":
												__UpperCamelCase    :    List[str]								=      "\n</w>"
									__UpperCamelCase    :    List[Any]								=      word
									return word

						def __lowerCamelCase					(				self , __UpperCamelCase , __UpperCamelCase="en" , __UpperCamelCase=False    )							->	Any:




									'''simple docstring'''


									__UpperCamelCase    :    Optional[int]								=      self.src_lang

									if self.do_lower_case:
												__UpperCamelCase    :    Optional[int]								=      text.lower()

									if bypass_tokenizer:
												__UpperCamelCase    :    List[Any]								=      text.split()
									else:
												__UpperCamelCase    :    Any								=      self.moses_pipeline(__UpperCamelCase , lang=__UpperCamelCase    )
												__UpperCamelCase    :    Dict								=      self.moses_tokenize(__UpperCamelCase , lang=__UpperCamelCase    )

									__UpperCamelCase    :    List[str]								=      []
									for token in text:
												if token:
															split_tokens.extend(list(self.bpe(__UpperCamelCase    ).split(" "    )    )    )

									return split_tokens

						def __lowerCamelCase					(				self , __UpperCamelCase    )							->	Union[str, Any]:




									'''simple docstring'''


									return self.encoder.get(__UpperCamelCase , self.encoder.get(self.unk_token    )    )

						def __lowerCamelCase					(				self , __UpperCamelCase    )							->	List[str]:




									'''simple docstring'''


									return self.decoder.get(__UpperCamelCase , self.unk_token    )

						def __lowerCamelCase					(				self , __UpperCamelCase    )							->	Any:




									'''simple docstring'''


									__UpperCamelCase    :    Optional[int]								=      [t.replace(" " , ""    ).replace("</w>" , " "    ) for t in tokens]
									__UpperCamelCase    :    Union[str, Any]								=      "".join(__UpperCamelCase    ).split()
									# detokenize
									__UpperCamelCase    :    Dict								=      self.moses_detokenize(__UpperCamelCase , self.tgt_lang    )
									return text

						def __lowerCamelCase					(				self , __UpperCamelCase , __UpperCamelCase = None    )							->	List[int]:




									'''simple docstring'''


									__UpperCamelCase    :    Union[str, Any]								=      [self.sep_token_id]

									# no bos used in fairseq
									if token_ids_a is None:
												return token_ids_a + sep
									return token_ids_a + sep + token_ids_a + sep

						def __lowerCamelCase					(				self , __UpperCamelCase , __UpperCamelCase = None , __UpperCamelCase = False    )							->	List[int]:




									'''simple docstring'''


									if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__UpperCamelCase , token_ids_a=__UpperCamelCase , already_has_special_tokens=__UpperCamelCase    )
									# no bos used in fairseq
									if token_ids_a is not None:
												return ([0] * len(__UpperCamelCase    )) + [1] + ([0] * len(__UpperCamelCase    )) + [1]
									return ([0] * len(__UpperCamelCase    )) + [1]

						def __lowerCamelCase					(				self , __UpperCamelCase , __UpperCamelCase = None    )							->	List[int]:




									'''simple docstring'''


									__UpperCamelCase    :    Dict								=      [self.sep_token_id]

									# no bos used in fairseq
									if token_ids_a is None:
												return len(token_ids_a + sep    ) * [0]
									return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]

						def __lowerCamelCase					(				self , __UpperCamelCase , __UpperCamelCase = None    )							->	Tuple[str]:




									'''simple docstring'''


									if not os.path.isdir(__UpperCamelCase    ):
												logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
												return

									__UpperCamelCase    :    Any								=      os.path.join(
									    __UpperCamelCase , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["src_vocab_file"]    )
									__UpperCamelCase    :    str								=      os.path.join(
									    __UpperCamelCase , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["tgt_vocab_file"]    )
									__UpperCamelCase    :    str								=      os.path.join(
									    __UpperCamelCase , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]    )

									with open(__UpperCamelCase , "w" , encoding="utf-8"    ) as f:
												f.write(json.dumps(self.encoder , indent=2 , sort_keys=__UpperCamelCase , ensure_ascii=__UpperCamelCase    ) + "\n"    )

									with open(__UpperCamelCase , "w" , encoding="utf-8"    ) as f:
												__UpperCamelCase    :    List[str]								=      {v: k for k, v in self.decoder.items()}
												f.write(json.dumps(__UpperCamelCase , indent=2 , sort_keys=__UpperCamelCase , ensure_ascii=__UpperCamelCase    ) + "\n"    )

									__UpperCamelCase    :    str								=      0
									with open(__UpperCamelCase , "w" , encoding="utf-8"    ) as writer:
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items() , key=lambda __UpperCamelCase    : kv[1]    ):
															if index != token_index:
																		logger.warning(
																		    f'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
																		    " Please check that the tokenizer is not corrupted!"    )
																		__UpperCamelCase    :    int								=      token_index
															writer.write(" ".join(__UpperCamelCase    ) + "\n"    )
															index += 1

									return src_vocab_file, tgt_vocab_file, merges_file

						def __getstate__(				self    )							->	Any:




									'''simple docstring'''


									__UpperCamelCase    :    Dict								=      self.__dict__.copy()
									__UpperCamelCase    :    Optional[int]								=      None
									return state




						def __setstate__(				self , __UpperCamelCase    )							->	List[str]:




									'''simple docstring'''


									__UpperCamelCase    :    int								=      d

									try:
												import sacremoses
									except ImportError:
												raise ImportError(
												    "You need to install sacremoses to use XLMTokenizer. "
												    "See https://pypi.org/project/sacremoses/ for installation."    )

									__UpperCamelCase    :    Union[str, Any]								=      sacremoses






lowercase     :       dict[str, float]        =       {
    "kilogram": 1,
    "gram": pow(10, 3),
    "milligram": pow(10, 6),
    "metric-ton": pow(10, -3),
    "long-ton": 0.0009842073,
    "short-ton": 0.0011023122,
    "pound": 2.2046244202,
    "stone": 0.1574731728,
    "ounce": 35.273990723,
    "carrat": 5000,
    "atomic-mass-unit": 6.0_22_13_66_52E26,
}

lowercase     :       dict[str, float]        =       {
    "kilogram": 1,
    "gram": pow(10, -3),
    "milligram": pow(10, -6),
    "metric-ton": pow(10, 3),
    "long-ton": 1016.04608,
    "short-ton": 907.184,
    "pound": 0.453592,
    "stone": 6.35029,
    "ounce": 0.0283495,
    "carrat": 0.0002,
    "atomic-mass-unit": 1.6_60_54_01_99E-27,
}







def  UpperCAmelCase_   (_lowerCAmelCase :		str     ,			_lowerCAmelCase :		str     ,			_lowerCAmelCase :		float		):
			if to_type not in KILOGRAM_CHART or from_type not in WEIGHT_TYPE_CHART:
						__UpperCamelCase    :    Dict								=      (
						    F'''Invalid \'from_type\' or \'to_type\' value: {from_type!r}, {to_type!r}\n'''
						    F'''Supported values are: {", ".join(_lowerCAmelCase		)}'''
						)
						raise ValueError(_lowerCAmelCase		)
			return value * KILOGRAM_CHART[to_type] * WEIGHT_TYPE_CHART[from_type]


if __name__ == "__main__":
					import doctest

					doctest.testmod()