



from typing import List

from packaging import version
from sklearn.metrics import fa_score

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse("""3.8"""):
      import importlib_metadata
else:
      import importlib.metadata as importlib_metadata


# TODO(Patrick/Anton)
_lowerCamelCase         =	"""\
"""

_lowerCamelCase         =	"""\
    XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages.
    XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval.
"""

_lowerCamelCase         =	"""
Compute XTREME-S evaluation metric associated to each XTREME-S dataset.
Args:
    predictions: list of predictions to score.
        Each translation should be tokenized into a list of tokens.
    references: list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
    bleu_kwargs: optional dict of keywords to be passed when computing 'bleu'.
        Keywords include Dict can be one of 'smooth_method', 'smooth_value', 'force', 'lowercase',
        'tokenize', 'use_effective_order'.
    wer_kwargs: optional dict of keywords to be passed when computing 'wer' and 'cer'.
        Keywords include 'concatenate_texts'.
Returns: depending on the XTREME-S task, one or several of:
    \"accuracy\": Accuracy - for 'fleurs-lang_id', 'minds14'
    \"f1\": F1 score - for 'minds14'
    \"wer\": Word error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel'
    \"cer\": Character error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel'
    \"bleu\": BLEU score according to the `sacrebleu` metric - for 'covost2'
Examples:

    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'mls')  # 'mls', 'voxpopuli', 'fleurs-asr' or 'babel'
    >>> references = [\"it is sunny here\", \"paper and pen are essentials\"]
    >>> predictions = [\"it's sunny\", \"paper pen are essential\"]
    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)
    >>> print({k: round(v, 2) for k, v in results.items()})
    {'wer': 0.56, 'cer': 0.27}

    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'covost2')
    >>> references = [\"bonjour paris\", \"il est necessaire de faire du sport de temps en temp\"]
    >>> predictions = [\"bonjour paris\", \"il est important de faire du sport souvent\"]
    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)
    >>> print({k: round(v, 2) for k, v in results.items()})
    {'bleu': 31.65}

    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'fleurs-lang_id')
    >>> references = [0, 1, 0, 0, 1]
    >>> predictions = [0, 1, 1, 0, 0]
    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)
    >>> print({k: round(v, 2) for k, v in results.items()})
    {'accuracy': 0.6}

    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'minds14')
    >>> references = [0, 1, 0, 0, 1]
    >>> predictions = [0, 1, 1, 0, 0]
    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)
    >>> print({k: round(v, 2) for k, v in results.items()})
    {'f1': 0.58, 'accuracy': 0.6}
"""

_lowerCamelCase         =	["""fleurs-asr""", """mls""", """voxpopuli""", """babel""", """covost2""", """fleurs-lang_id""", """minds14"""]
_lowerCamelCase         =	""""""

try:
      from jiwer import transforms as tr

      _lowerCamelCase         =	True
except ImportError:
      _lowerCamelCase         =	False

if _jiwer_available and version.parse(importlib_metadata.version("""jiwer""")) < version.parse("""2.3.0"""):

      class    _SCREAMING_SNAKE_CASE						(tr.AbstractTransform			):





           def __init__(							self			:				Tuple  ,	UpperCamelCase			:				str = " "					)->List[str]:
            __SCREAMING_SNAKE_CASE							:     List[Any]							=						sentence_delimiter





           def 						__snake_case     (							self			:				Union[str, Any]  ,	UpperCamelCase			:				str					)->Any:
            return list(UpperCamelCase					)






           def 						__snake_case     (							self			:				str  ,	UpperCamelCase			:				List[str]					)->Tuple:
            __SCREAMING_SNAKE_CASE							:     int							=						[]
            for sent_idx, sentence in enumerate(UpperCamelCase					):
             chars.extend(self.process_string(UpperCamelCase					)					)
             if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(UpperCamelCase					) - 1:
              chars.append(self.sentence_delimiter					)
            return chars






      _lowerCamelCase         =	tr.Compose(
          [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
      )
elif _jiwer_available:
      _lowerCamelCase         =	tr.Compose(
          [
              tr.RemoveMultipleSpaces(),
              tr.Strip(),
              tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
              tr.ReduceToListOfListOfChars(),
          ]
      )
else:
      _lowerCamelCase         =	None






def 						_lowerCAmelCase    (					__lowerCamelCase :  Optional[Any]		,						__lowerCamelCase :  Union[str, Any]  ):



 """simple docstring"""





 return float((preds == labels).mean()  )






def 						_lowerCAmelCase    (					__lowerCamelCase :  Dict		,						__lowerCamelCase :  str  ):



 """simple docstring"""





 return {
     "f1": float(fa_score(y_true=__lowerCamelCase		,						y_pred=__lowerCamelCase		,						average="macro"  )  ),
     "accuracy": simple_accuracy(__lowerCamelCase		,						__lowerCamelCase  ),
 }






def 						_lowerCAmelCase    (					__lowerCamelCase :  str		,						__lowerCamelCase :  List[str]		,						__lowerCamelCase :  Optional[int]="exp"		,						__lowerCamelCase :  Optional[Any]=None		,						__lowerCamelCase :  Tuple=False		,						__lowerCamelCase :  Dict=False		,						__lowerCamelCase :  Dict=None		,						__lowerCamelCase :  Optional[Any]=False		,						):



 """simple docstring"""





 __SCREAMING_SNAKE_CASE							:     Tuple							=						[[label] for label in labels]
 __SCREAMING_SNAKE_CASE							:     List[Any]							=						list(__lowerCamelCase  )
 try:
  import sacrebleu as scb
 except ImportError:
  raise ValueError(
      "sacrebleu has to be installed in order to apply the bleu metric for covost2."
      "You can install it via `pip install sacrebleu`."  )

 if version.parse(scb.__version__  ) < version.parse("1.4.12"  ):
  raise ImportWarning(
      "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"
      "You can install it with `pip install \"sacrebleu>=1.4.12\"`."  )

 __SCREAMING_SNAKE_CASE							:     Optional[int]							=						len(labels[0]  )
 if any(len(__lowerCamelCase  ) != references_per_prediction for refs in labels  ):
  raise ValueError("Sacrebleu requires the same number of references for each prediction"  )
 __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						[[refs[i] for refs in labels] for i in range(__lowerCamelCase  )]
 __SCREAMING_SNAKE_CASE							:     List[Any]							=						scb.corpus_bleu(
     __lowerCamelCase		,						__lowerCamelCase		,						smooth_method=__lowerCamelCase		,						smooth_value=__lowerCamelCase		,						force=__lowerCamelCase		,						lowercase=__lowerCamelCase		,						use_effective_order=__lowerCamelCase		,						**({"tokenize": tokenize} if tokenize else {})		,						)
 return {"bleu": output.score}






def 						_lowerCAmelCase    (					__lowerCamelCase :  List[str]		,						__lowerCamelCase :  List[str]		,						__lowerCamelCase :  Union[str, Any]		,						__lowerCamelCase :  int  ):



 """simple docstring"""





 try:
  from jiwer import compute_measures
 except ImportError:
  raise ValueError(
      F"""jiwer has to be installed in order to apply the wer metric for {config_name}."""
      "You can install it via `pip install jiwer`."  )

 if concatenate_texts:
  __SCREAMING_SNAKE_CASE							:     Dict							=						compute_measures(__lowerCamelCase		,						__lowerCamelCase  )["wer"]

  __SCREAMING_SNAKE_CASE							:     List[Any]							=						compute_measures(__lowerCamelCase		,						__lowerCamelCase		,						truth_transform=__lowerCamelCase		,						hypothesis_transform=__lowerCamelCase  )["wer"]
  return {"wer": wer, "cer": cer}
 else:

  def compute_score(__lowerCamelCase :  Any		,						__lowerCamelCase :  Optional[int]		,						__lowerCamelCase :  List[Any]="wer"  ):
   __SCREAMING_SNAKE_CASE							:     List[Any]							=						0
   __SCREAMING_SNAKE_CASE							:     Optional[int]							=						0
   for prediction, reference in zip(__lowerCamelCase		,						__lowerCamelCase  ):
    if score_type == "wer":
     __SCREAMING_SNAKE_CASE							:     List[str]							=						compute_measures(__lowerCamelCase		,						__lowerCamelCase  )
    elif score_type == "cer":
     __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						compute_measures(
         __lowerCamelCase		,						__lowerCamelCase		,						truth_transform=__lowerCamelCase		,						hypothesis_transform=__lowerCamelCase  )
    incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
    total += measures["substitutions"] + measures["deletions"] + measures["hits"]
   return incorrect / total

  return {"wer": compute_score(__lowerCamelCase		,						__lowerCamelCase		,						"wer"  ), "cer": compute_score(__lowerCamelCase		,						__lowerCamelCase		,						"cer"  )}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION      ,     _KWARGS_DESCRIPTION			)
class    _SCREAMING_SNAKE_CASE						(datasets.Metric			):





     def 						__snake_case     (							self			:				Optional[Any]					)->Tuple:
      if self.config_name not in _CONFIG_NAMES:
       raise KeyError(F"""You should supply a configuration name selected in {_CONFIG_NAMES}"""					)

      __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						"int64" if self.config_name in ["fleurs-lang_id", "minds14"] else "string"

      return datasets.MetricInfo(
          description=_DESCRIPTION  ,	citation=_CITATION  ,	inputs_description=_KWARGS_DESCRIPTION  ,	features=datasets.Features(
              {"predictions": datasets.Value(UpperCamelCase					), "references": datasets.Value(UpperCamelCase					)}					)  ,	codebase_urls=[]  ,	reference_urls=[]  ,	format="numpy"  ,	)






     def 						__snake_case     (							self			:				List[Any]  ,	UpperCamelCase			:				Union[str, Any]  ,	UpperCamelCase			:				List[str]  ,	UpperCamelCase			:				Union[str, Any]=None  ,	UpperCamelCase			:				Dict=None					)->Optional[int]:
      __SCREAMING_SNAKE_CASE							:     str							=						bleu_kwargs if bleu_kwargs is not None else {}
      __SCREAMING_SNAKE_CASE							:     Any							=						wer_kwargs if wer_kwargs is not None else {}

      if self.config_name == "fleurs-lang_id":
       return {"accuracy": simple_accuracy(UpperCamelCase  ,	UpperCamelCase					)}
      elif self.config_name == "minds14":
       return fa_and_simple_accuracy(UpperCamelCase  ,	UpperCamelCase					)
      elif self.config_name == "covost2":
       __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						bleu_kwargs.pop("smooth_method"  ,	"exp"					)
       __SCREAMING_SNAKE_CASE							:     List[str]							=						bleu_kwargs.pop("smooth_value"  ,	UpperCamelCase					)
       __SCREAMING_SNAKE_CASE							:     str							=						bleu_kwargs.pop("force"  ,	UpperCamelCase					)
       __SCREAMING_SNAKE_CASE							:     Tuple							=						bleu_kwargs.pop("lowercase"  ,	UpperCamelCase					)
       __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						bleu_kwargs.pop("tokenize"  ,	UpperCamelCase					)
       __SCREAMING_SNAKE_CASE							:     Dict							=						bleu_kwargs.pop("use_effective_order"  ,	UpperCamelCase					)
       return bleu(
           preds=UpperCamelCase  ,	labels=UpperCamelCase  ,	smooth_method=UpperCamelCase  ,	smooth_value=UpperCamelCase  ,	force=UpperCamelCase  ,	lowercase=UpperCamelCase  ,	tokenize=UpperCamelCase  ,	use_effective_order=UpperCamelCase  ,	)
      elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]:
       __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						wer_kwargs.pop("concatenate_texts"  ,	UpperCamelCase					)
       return wer_and_cer(UpperCamelCase  ,	UpperCamelCase  ,	UpperCamelCase  ,	self.config_name					)
      else:
       raise KeyError(F"""You should supply a configuration name selected in {_CONFIG_NAMES}"""					)










'''simple docstring'''






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


a_       :     Tuple 							=     logging.get_logger(__name__)

a_       :     Optional[Any] 							=     {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

a_       :     str 							=     {
    "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
    "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
}

a_       :     Union[str, Any] 							=     {
    "openai-gpt": 5_1_2,
}






def 						_A					(lowerCAmelCase__	:Any		) ->	int:




     '''simple docstring'''



     _a         =				text.strip()
     if not text:
          return []
     _a         =				text.split()
     return tokens




class 			a  (  _SCREAMING_SNAKE_CASE							):
 def __init__(			self   ,					__magic_name__=True   ,					__magic_name__=None   ,					__magic_name__=True   ,					__magic_name__=None   ,					__magic_name__=True   ,					)       ->				Optional[Any]:
      if never_split is None:
           _a         =				[]
      _a         =				do_lower_case
      _a         =				set(__magic_name__    )
      _a         =				tokenize_chinese_chars
      _a         =				strip_accents
      _a         =				do_split_on_punc
 def 				__UpperCAmelCase		(			self   ,					__magic_name__   ,					__magic_name__=None    )       ->				List[str]:
      _a         =				self.never_split.union(set(__magic_name__    )    ) if never_split else self.never_split
      _a         =				self._clean_text(__magic_name__    )

      # This was added on November 1st, 2018 for the multilingual and Chinese
      # models. This is also applied to the English models now, but it doesn't
      # matter since the English models were not trained on any Chinese data
      # and generally don't have any Chinese data in them (there are Chinese
      # characters in the vocabulary because Wikipedia does have some Chinese
      # words in the English Wikipedia.).
      if self.tokenize_chinese_chars:
           _a         =				self._tokenize_chinese_chars(__magic_name__    )
      # prevents treating the same character with different unicode codepoints as different characters
      _a         =				unicodedata.normalize('NFC'   ,					__magic_name__    )
      _a         =				whitespace_tokenize(__magic_name__    )
      _a         =				[]
      for token in orig_tokens:
           if token not in never_split:
                if self.do_lower_case:
                     _a         =				token.lower()
                     if self.strip_accents is not False:
                          _a         =				self._run_strip_accents(__magic_name__    )
                elif self.strip_accents:
                     _a         =				self._run_strip_accents(__magic_name__    )
           split_tokens.extend(self._run_split_on_punc(__magic_name__   ,					__magic_name__    )    )

      _a         =				whitespace_tokenize(' '.join(__magic_name__    )    )
      return output_tokens
 def 				__UpperCAmelCase		(			self   ,					__magic_name__    )       ->				List[str]:
      _a         =				unicodedata.normalize('NFD'   ,					__magic_name__    )
      _a         =				[]
      for char in text:
           _a         =				unicodedata.category(__magic_name__    )
           if cat == "Mn":
                continue
           output.append(__magic_name__    )
      return "".join(__magic_name__    )
 def 				__UpperCAmelCase		(			self   ,					__magic_name__   ,					__magic_name__=None    )       ->				Dict:
      if not self.do_split_on_punc or (never_split is not None and text in never_split):
           return [text]
      _a         =				list(__magic_name__    )
      _a         =				0
      _a         =				True
      _a         =				[]
      while i < len(__magic_name__    ):
           _a         =				chars[i]
           if _is_punctuation(__magic_name__    ):
                output.append([char]    )
                _a         =				True
           else:
                if start_new_word:
                     output.append([]    )
                _a         =				False
                output[-1].append(__magic_name__    )
           i += 1

      return ["".join(__magic_name__    ) for x in output]
 def 				__UpperCAmelCase		(			self   ,					__magic_name__    )       ->				Tuple:
      _a         =				[]
      for char in text:
           _a         =				ord(__magic_name__    )
           if self._is_chinese_char(__magic_name__    ):
                output.append(' '    )
                output.append(__magic_name__    )
                output.append(' '    )
           else:
                output.append(__magic_name__    )
      return "".join(__magic_name__    )
 def 				__UpperCAmelCase		(			self   ,					__magic_name__    )       ->				Union[str, Any]:
      # This defines a "chinese character" as anything in the CJK Unicode block:
      #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
      #
      # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
      # despite its name. The modern Korean Hangul alphabet is a different block,
      # as is Japanese Hiragana and Katakana. Those alphabets are used to write
      # space-separated words, so they are not treated specially and handled
      # like the all of the other languages.
      if (
          (cp >= 0x4E00 and cp <= 0x9FFF)
          or (cp >= 0x3400 and cp <= 0x4DBF)  #
          or (cp >= 0x2_0000 and cp <= 0x2_A6DF)  #
          or (cp >= 0x2_A700 and cp <= 0x2_B73F)  #
          or (cp >= 0x2_B740 and cp <= 0x2_B81F)  #
          or (cp >= 0x2_B820 and cp <= 0x2_CEAF)  #
          or (cp >= 0xF900 and cp <= 0xFAFF)
          or (cp >= 0x2_F800 and cp <= 0x2_FA1F)  #
      ):  #
           return True

      return False




 def 				__UpperCAmelCase		(			self   ,					__magic_name__    )       ->				Any:
      _a         =				[]
      for char in text:
           _a         =				ord(__magic_name__    )
           if cp == 0 or cp == 0xFFFD or _is_control(__magic_name__    ):
                continue
           if _is_whitespace(__magic_name__    ):
                output.append(' '    )
           else:
                output.append(__magic_name__    )
      return "".join(__magic_name__    )






def 						_A					(lowerCAmelCase__	:str		) ->	Tuple:




     '''simple docstring'''



     _a         =				set()
     _a         =				word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)		)
          _a         =				char
     return pairs






def 						_A					(lowerCAmelCase__	:Tuple		) ->	Union[str, Any]:




     '''simple docstring'''



     _a         =				text.replace('—'					,    '-'		)
     _a         =				text.replace('–'					,    '-'		)
     _a         =				text.replace('―'					,    '-'		)
     _a         =				text.replace('…'					,    '...'		)
     _a         =				text.replace('´'					,    '\''		)
     _a         =				re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'					,    r' \1 '					,    lowerCAmelCase__		)
     _a         =				re.sub(r'\s*\n\s*'					,    ' \n '					,    lowerCAmelCase__		)
     _a         =				re.sub(r'[^\S\n]+'					,    ' '					,    lowerCAmelCase__		)
     return text.strip()






class 			a  (  _SCREAMING_SNAKE_CASE							):
 _lowerCAmelCase			       =						VOCAB_FILES_NAMES
 _lowerCAmelCase			       =						PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase			       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase			       =						["""input_ids""", """attention_mask"""]
 def __init__(			self   ,					__magic_name__   ,					__magic_name__   ,					__magic_name__="<unk>"   ,					**__magic_name__    )       ->				Any:
      super().__init__(unk_token=__magic_name__   ,					**__magic_name__    )

      try:
           import ftfy
           from spacy.lang.en import English

           _a         =				English()
           _a         =				_nlp.tokenizer
           _a         =				ftfy.fix_text
      except ImportError:
           logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'    )
           _a         =				BasicTokenizer(do_lower_case=__magic_name__    )
           _a         =				None

      with open(__magic_name__   ,					encoding='utf-8'    ) as vocab_handle:
           _a         =				json.load(__magic_name__    )
      _a         =				{v: k for k, v in self.encoder.items()}
      with open(__magic_name__   ,					encoding='utf-8'    ) as merges_handle:
           _a         =				merges_handle.read().split('\n'    )[1:-1]
      _a         =				[tuple(merge.split()    ) for merge in merges]
      _a         =				dict(zip(__magic_name__   ,					range(len(__magic_name__    )    )    )    )
      _a         =				{}
 @property
 def 				__UpperCAmelCase		(			self    )       ->				Optional[int]:
      return True
 @property
 def 				__UpperCAmelCase		(			self    )       ->				int:
      return len(self.encoder    )
 def 				__UpperCAmelCase		(			self    )       ->				List[str]:
      return dict(self.encoder   ,					**self.added_tokens_encoder    )
 def 				__UpperCAmelCase		(			self   ,					__magic_name__    )       ->				Any:
      _a         =				tuple(token[:-1]    ) + (token[-1] + '</w>',)
      if token in self.cache:
           return self.cache[token]
      _a         =				get_pairs(__magic_name__    )

      if not pairs:
           return token + "</w>"

      while True:
           _a         =				min(__magic_name__   ,					key=lambda __magic_name__    : self.bpe_ranks.get(__magic_name__   ,					float('inf'    )    )    )
           if bigram not in self.bpe_ranks:
                break
           _a   , _a         =				bigram
           _a         =				[]
           _a         =				0
           while i < len(__magic_name__    ):
                try:
                     _a         =				word.index(__magic_name__   ,					__magic_name__    )
                except ValueError:
                     new_word.extend(word[i:]    )
                     break
                else:
                     new_word.extend(word[i:j]    )
                     _a         =				j

                if word[i] == first and i < len(__magic_name__    ) - 1 and word[i + 1] == second:
                     new_word.append(first + second    )
                     i += 2
                else:
                     new_word.append(word[i]    )
                     i += 1
           _a         =				tuple(__magic_name__    )
           _a         =				new_word
           if len(__magic_name__    ) == 1:
                break
           else:
                _a         =				get_pairs(__magic_name__    )
      _a         =				' '.join(__magic_name__    )
      if word == "\n  </w>":
           _a         =				'\n</w>'
      _a         =				word
      return word
 def 				__UpperCAmelCase		(			self   ,					__magic_name__    )       ->				Union[str, Any]:
      _a         =				[]
      if self.fix_text is None:
           # Using BERT's BasicTokenizer
           _a         =				self.nlp.tokenize(__magic_name__    )
           for token in text:
                split_tokens.extend(list(self.bpe(__magic_name__    ).split(' '    )    )    )
      else:
           # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
           _a         =				self.nlp(text_standardize(self.fix_text(__magic_name__    )    )    )
           for token in text:
                split_tokens.extend(list(self.bpe(token.text.lower()    ).split(' '    )    )    )
      return split_tokens
 def 				__UpperCAmelCase		(			self   ,					__magic_name__    )       ->				Any:
      return self.encoder.get(__magic_name__   ,					self.encoder.get(self.unk_token    )    )
 def 				__UpperCAmelCase		(			self   ,					__magic_name__    )       ->				Any:
      return self.decoder.get(__magic_name__   ,					self.unk_token    )
 def 				__UpperCAmelCase		(			self   ,					__magic_name__    )       ->				Dict:
      _a         =				''.join(__magic_name__    ).replace('</w>'   ,					' '    ).strip()
      return out_string




 def 				__UpperCAmelCase		(			self   ,					__magic_name__   ,					__magic_name__ = None    )       ->				Tuple[str]:
      if not os.path.isdir(__magic_name__    ):
           logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
           return
      _a         =				os.path.join(
          __magic_name__   ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
      _a         =				os.path.join(
          __magic_name__   ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']    )

      with open(__magic_name__   ,					'w'   ,					encoding='utf-8'    ) as f:
           f.write(json.dumps(self.encoder   ,					indent=2   ,					sort_keys=__magic_name__   ,					ensure_ascii=__magic_name__    ) + '\n'    )

      _a         =				0
      with open(__magic_name__   ,					'w'   ,					encoding='utf-8'    ) as writer:
           writer.write('#version: 0.2\n'    )
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,					key=lambda __magic_name__    : kv[1]    ):
                if index != token_index:
                     logger.warning(
                         f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                         ' Please check that the tokenizer is not corrupted!'    )
                     _a         =				token_index
                writer.write(' '.join(__magic_name__    ) + '\n'    )
                index += 1

      return vocab_file, merge_file

