



"""simple docstring"""




import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__			=   logging.get_logger(__name__)

lowerCamelCase__			=   {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCamelCase__			=   {
    'vocab_file': {
        'microsoft/biogpt': 'https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json',
    },
    'merges_file': {'microsoft/biogpt': 'https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt'},
}

lowerCamelCase__			=   {
    'microsoft/biogpt': 1_024,
}






def 						_SCREAMING_SNAKE_CASE							( UpperCamelCase     :	Tuple    ):
  A__											=							set()
  A__											=							word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)    )
    A__											=							char
  return pairs







class 					_UpperCamelCase		( lowercase_):
    __lowerCamelCase         =  VOCAB_FILES_NAMES
    __lowerCamelCase         =  PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase         =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowerCamelCase         =  ["""input_ids""", """attention_mask"""]






    def __init__(self						,      lowerCamelCase__						,      lowerCamelCase__						,      lowerCamelCase__="<unk>"						,      lowerCamelCase__="<s>"						,      lowerCamelCase__="</s>"						,      lowerCamelCase__="</s>"						,      lowerCamelCase__="<pad>"						,      **lowerCamelCase__						,      ):


      """simple docstring"""
      super().__init__(
          bos_token=lowerCamelCase__						,      eos_token=lowerCamelCase__						,      sep_token=lowerCamelCase__						,      unk_token=lowerCamelCase__						,      pad_token=lowerCamelCase__						,      **lowerCamelCase__						,      )

      try:
        import sacremoses
      except ImportError:
        raise ImportError(
            """You need to install sacremoses to use BioGptTokenizer. """
            """See https://pypi.org/project/sacremoses/ for installation."""   )

      A__											=							"""en"""
      A__											=							sacremoses
      # cache of sm.MosesTokenizer instance
      A__											=							{}
      A__											=							{}
      with open(lowerCamelCase__						,      encoding="""utf-8"""   ) as vocab_handle:
        A__											=							json.load(lowerCamelCase__   )
      A__											=							{v: k for k, v in self.encoder.items()}
      with open(lowerCamelCase__						,      encoding="""utf-8"""   ) as merges_handle:
        A__											=							merges_handle.read().split("""\n"""   )[:-1]
      A__											=							[tuple(merge.split()[:2]   ) for merge in merges]
      A__											=							dict(zip(lowerCamelCase__						,      range(len(lowerCamelCase__   )   )   )   )
      A__											=							{}






    @property
    def  A    (self   ):


      """simple docstring"""
      return len(self.encoder   )






    def  A    (self   ):


      """simple docstring"""
      return dict(self.encoder						,      **self.added_tokens_encoder   )






    def  A    (self						,      lowerCamelCase__						,      lowerCamelCase__   ):


      """simple docstring"""







      if lang not in self.cache_moses_tokenizer:
        A__											=							self.sm.MosesTokenizer(lang=lowerCamelCase__   )
        A__											=							moses_tokenizer
      return self.cache_moses_tokenizer[lang].tokenize(
          lowerCamelCase__						,      aggressive_dash_splits=lowerCamelCase__						,      return_str=lowerCamelCase__						,      escape=lowerCamelCase__   )






    def  A    (self						,      lowerCamelCase__						,      lowerCamelCase__   ):


      """simple docstring"""







      if lang not in self.cache_moses_detokenizer:
        A__											=							self.sm.MosesDetokenizer(lang=lowerCamelCase__   )
        A__											=							moses_detokenizer
      return self.cache_moses_detokenizer[lang].detokenize(lowerCamelCase__   )






    def  A    (self						,      lowerCamelCase__   ):


      """simple docstring"""
      A__											=							tuple(token[:-1]   ) + (token[-1] + """</w>""",)
      if token in self.cache:
        return self.cache[token]
      A__											=							get_pairs(lowerCamelCase__   )

      if not pairs:
        return token + "</w>"

      while True:
        A__											=							min(lowerCamelCase__						,      key=lambda lowerCamelCase__   : self.bpe_ranks.get(lowerCamelCase__						,      float("""inf"""   )   )   )
        if bigram not in self.bpe_ranks:
          break
        A__						,A__											=							bigram
        A__											=							[]
        A__											=							0
        while i < len(lowerCamelCase__   ):
          try:
            A__											=							word.index(lowerCamelCase__						,      lowerCamelCase__   )
          except ValueError:
            new_word.extend(word[i:]   )
            break
          else:
            new_word.extend(word[i:j]   )
            A__											=							j

          if word[i] == first and i < len(lowerCamelCase__   ) - 1 and word[i + 1] == second:
            new_word.append(first + second   )
            i += 2
          else:
            new_word.append(word[i]   )
            i += 1
        A__											=							tuple(lowerCamelCase__   )
        A__											=							new_word
        if len(lowerCamelCase__   ) == 1:
          break
        else:
          A__											=							get_pairs(lowerCamelCase__   )
      A__											=							""" """.join(lowerCamelCase__   )
      if word == "\n  </w>":
        A__											=							"""\n</w>"""
      A__											=							word
      return word






    def  A    (self						,      lowerCamelCase__						,      lowerCamelCase__=False   ):


      """simple docstring"""







      if bypass_tokenizer:
        A__											=							text.split()
      else:
        A__											=							self.moses_tokenize(lowerCamelCase__						,      self.lang   )

      A__											=							[]
      for token in text:
        if token:
          split_tokens.extend(list(self.bpe(lowerCamelCase__   ).split(""" """   )   )   )

      return split_tokens






    def  A    (self						,      lowerCamelCase__   ):


      """simple docstring"""
      return self.encoder.get(lowerCamelCase__						,      self.encoder.get(self.unk_token   )   )






    def  A    (self						,      lowerCamelCase__   ):


      """simple docstring"""
      return self.decoder.get(lowerCamelCase__						,      self.unk_token   )






    def  A    (self						,      lowerCamelCase__   ):


      """simple docstring"""
      A__											=							[t.replace(""" """						,      """"""   ).replace("""</w>"""						,      """ """   ) for t in tokens]
      A__											=							"""""".join(lowerCamelCase__   ).split()
      # detokenize
      A__											=							self.moses_detokenize(lowerCamelCase__						,      self.lang   )
      return text






    def  A    (self						,      lowerCamelCase__						,      lowerCamelCase__ = None   ):


      """simple docstring"""







      if token_ids_a is None:
        return [self.sep_token_id] + token_ids_a
      A__											=							[self.sep_token_id]
      return sep + token_ids_a + sep + token_ids_a






    def  A    (self						,      lowerCamelCase__						,      lowerCamelCase__ = None						,      lowerCamelCase__ = False   ):


      """simple docstring"""







      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=lowerCamelCase__						,      token_ids_a=lowerCamelCase__						,      already_has_special_tokens=lowerCamelCase__   )
      # no bos used in fairseq
      if token_ids_a is not None:
        return [1] + ([0] * len(lowerCamelCase__   )) + [1] + ([0] * len(lowerCamelCase__   ))
      return [1] + ([0] * len(lowerCamelCase__   ))






    def  A    (self						,      lowerCamelCase__						,      lowerCamelCase__ = None   ):


      """simple docstring"""
      A__											=							[self.sep_token_id]

      # no bos used in fairseq
      if token_ids_a is None:
        return len(token_ids_a + sep   ) * [0]
      return len(token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






    def  A    (self						,      lowerCamelCase__						,      lowerCamelCase__ = None   ):


      """simple docstring"""







      if not os.path.isdir(lowerCamelCase__   ):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
        return
      A__											=							os.path.join(
          lowerCamelCase__						,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
      A__											=							os.path.join(
          lowerCamelCase__						,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

      with open(lowerCamelCase__						,      """w"""						,      encoding="""utf-8"""   ) as f:
        f.write(json.dumps(self.encoder						,      indent=2						,      sort_keys=lowerCamelCase__						,      ensure_ascii=lowerCamelCase__   ) + """\n"""   )

      A__											=							0
      with open(lowerCamelCase__						,      """w"""						,      encoding="""utf-8"""   ) as writer:
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,      key=lambda lowerCamelCase__   : kv[1]   ):
          if index != token_index:
            logger.warning(
                F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                """ Please check that the tokenizer is not corrupted!"""   )
            A__											=							token_index
          writer.write(""" """.join(lowerCamelCase__   ) + """\n"""   )
          index += 1

      return vocab_file, merge_file






    def __getstate__(self   ):


      """simple docstring"""
      A__											=							self.__dict__.copy()
      A__											=							None
      return state






    def __setstate__(self						,      lowerCamelCase__   ):


      """simple docstring"""
      A__											=							d

      try:
        import sacremoses
      except ImportError:
        raise ImportError(
            """You need to install sacremoses to use XLMTokenizer. """
            """See https://pypi.org/project/sacremoses/ for installation."""   )

      A__											=							sacremoses












"""simple docstring"""




def 						_SCREAMING_SNAKE_CASE							( UpperCamelCase     :	int		,				UpperCamelCase     :	int = 1000    ):
  if (not isinstance(UpperCamelCase		,				UpperCamelCase    )) or (digit_position <= 0):
    raise ValueError("""Digit position must be a positive integer"""    )
  elif (not isinstance(UpperCamelCase		,				UpperCamelCase    )) or (precision < 0):
    raise ValueError("""Precision must be a nonnegative integer"""    )

  # compute an approximation of (16 ** (n - 1)) * pi whose fractional part is mostly
  # accurate
  A__											=							(
      4 * _subsum(UpperCamelCase		,				1		,				UpperCamelCase    )
      - 2 * _subsum(UpperCamelCase		,				4		,				UpperCamelCase    )
      - _subsum(UpperCamelCase		,				5		,				UpperCamelCase    )
      - _subsum(UpperCamelCase		,				6		,				UpperCamelCase    )
  )

  # return the first hex digit of the fractional part of the result
  return hex(int((sum_result % 1) * 16    )    )[2:]






def 						_SCREAMING_SNAKE_CASE							( UpperCamelCase     :	int		,				UpperCamelCase     :	int		,				UpperCamelCase     :	int    ):
  A__											=							0.0
  for sum_index in range(digit_pos_to_extract + precision    ):
    A__											=							8 * sum_index + denominator_addend
    if sum_index < digit_pos_to_extract:
      # if the exponential term is an integer and we mod it by the denominator
      # before dividing, only the integer part of the sum will change;
      # the fractional part will not
      A__											=							pow(
          16		,				digit_pos_to_extract - 1 - sum_index		,				UpperCamelCase    )
    else:
      A__											=							pow(16		,				digit_pos_to_extract - 1 - sum_index    )
    total += exponential_term / denominator
  return total


if __name__ == "__main__":
      import doctest

      doctest.testmod()




