


'''simple docstring'''


def 		_lowercase   (SCREAMING_SNAKE_CASE       ):




     '''simple docstring'''





     if len(SCREAMING_SNAKE_CASE       ) == 1:
          return a[0], a[0]
     else:
          # split A into half.
          __A  :					Optional[Any]						     =       a[: len(SCREAMING_SNAKE_CASE       ) // 2]
          __A  :					str						     =       a[len(SCREAMING_SNAKE_CASE       ) // 2 :]

          # 2 sub problems, 1/2 of original size.
          __A     ,__A  :					str						     =       max_difference(SCREAMING_SNAKE_CASE       )
          __A     ,__A  :					Optional[Any]						     =       max_difference(SCREAMING_SNAKE_CASE       )

          # get min of first and max of second
          # linear time
          __A  :					Optional[Any]						     =       min(SCREAMING_SNAKE_CASE       )
          __A  :					Union[str, Any]						     =       max(SCREAMING_SNAKE_CASE       )

          # 3 cases, either (small1, big1),
          # (min_first, max_second), (small2, big2)
          # constant comparisons
          if biga - smalla > max_second - min_first and biga - smalla > biga - smalla:
               return smalla, biga
          elif biga - smalla > max_second - min_first:
               return smalla, biga
          else:
               return min_first, max_second


if __name__ == "__main__":
      import doctest

      doctest.testmod()





'''simple docstring'''


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_UpperCamelCase          =     logging.get_logger(__name__)

_UpperCamelCase          =     {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

_UpperCamelCase          =     {
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

_UpperCamelCase          =     {
    """openai-gpt""": 512,
}


def 		_lowercase   (SCREAMING_SNAKE_CASE       ):




     '''simple docstring'''





     __A  :					List[str]						     =       text.strip()
     if not text:
          return []
     __A  :					Any						     =       text.split()
     return tokens







class        __magic_name__  (					lowerCAmelCase   ):

     """simple docstring"""






     def __init__(      self ,    lowerCamelCase=True ,    lowerCamelCase=None ,    lowerCamelCase=True ,    lowerCamelCase=None ,    lowerCamelCase=True ,    ):


          '''simple docstring'''





          if never_split is None:
               __A  :					List[Any]						     =       []
          __A  :					str						     =       do_lower_case
          __A  :					str						     =       set(lowerCamelCase		)
          __A  :					Tuple						     =       tokenize_chinese_chars
          __A  :					Any						     =       strip_accents
          __A  :					Dict						     =       do_split_on_punc


     def lowerCAmelCase__     (      self ,    lowerCamelCase ,    lowerCamelCase=None		):


          '''simple docstring'''





          __A  :					str						     =       self.never_split.union(set(lowerCamelCase		)		) if never_split else self.never_split
          __A  :					Optional[Any]						     =       self._clean_text(lowerCamelCase		)

          # This was added on November 1st, 2018 for the multilingual and Chinese
          # models. This is also applied to the English models now, but it doesn't
          # matter since the English models were not trained on any Chinese data
          # and generally don't have any Chinese data in them (there are Chinese
          # characters in the vocabulary because Wikipedia does have some Chinese
          # words in the English Wikipedia.).
          if self.tokenize_chinese_chars:
               __A  :					Tuple						     =       self._tokenize_chinese_chars(lowerCamelCase		)
          # prevents treating the same character with different unicode codepoints as different characters
          __A  :					str						     =       unicodedata.normalize("NFC" ,    lowerCamelCase		)
          __A  :					Tuple						     =       whitespace_tokenize(lowerCamelCase		)
          __A  :					Union[str, Any]						     =       []
          for token in orig_tokens:
               if token not in never_split:
                    if self.do_lower_case:
                         __A  :					str						     =       token.lower()
                         if self.strip_accents is not False:
                              __A  :					str						     =       self._run_strip_accents(lowerCamelCase		)
                    elif self.strip_accents:
                         __A  :					Tuple						     =       self._run_strip_accents(lowerCamelCase		)
               split_tokens.extend(self._run_split_on_punc(lowerCamelCase ,    lowerCamelCase		)		)

          __A  :					Tuple						     =       whitespace_tokenize(" ".join(lowerCamelCase		)		)
          return output_tokens


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					Tuple						     =       unicodedata.normalize("NFD" ,    lowerCamelCase		)
          __A  :					Tuple						     =       []
          for char in text:
               __A  :					Dict						     =       unicodedata.category(lowerCamelCase		)
               if cat == "Mn":
                    continue
               output.append(lowerCamelCase		)
          return "".join(lowerCamelCase		)


     def lowerCAmelCase__     (      self ,    lowerCamelCase ,    lowerCamelCase=None		):


          '''simple docstring'''





          if not self.do_split_on_punc or (never_split is not None and text in never_split):
               return [text]
          __A  :					int						     =       list(lowerCamelCase		)
          __A  :					int						     =       0
          __A  :					int						     =       True
          __A  :					Tuple						     =       []
          while i < len(lowerCamelCase		):
               __A  :					str						     =       chars[i]
               if _is_punctuation(lowerCamelCase		):
                    output.append([char]		)
                    __A  :					Optional[int]						     =       True
               else:
                    if start_new_word:
                         output.append([]		)
                    __A  :					List[str]						     =       False
                    output[-1].append(lowerCamelCase		)
               i += 1

          return ["".join(lowerCamelCase		) for x in output]


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					int						     =       []
          for char in text:
               __A  :					Union[str, Any]						     =       ord(lowerCamelCase		)
               if self._is_chinese_char(lowerCamelCase		):
                    output.append(" "		)
                    output.append(lowerCamelCase		)
                    output.append(" "		)
               else:
                    output.append(lowerCamelCase		)
          return "".join(lowerCamelCase		)


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          if (
              (cp >= 0x4E00 and cp <= 0x9FFF)
              or (cp >= 0x3400 and cp <= 0x4DBF)  #
              or (cp >= 0x2_0000 and cp <= 0x2_A6DF)  #
              or (cp >= 0x2_A700 and cp <= 0x2_B73F)  #
              or (cp >= 0x2_B740 and cp <= 0x2_B81F)  #
              or (cp >= 0x2_B820 and cp <= 0x2_CEAF)  #
              or (cp >= 0xF900 and cp <= 0xFAFF)
              or (cp >= 0x2_F800 and cp <= 0x2_FA1F)  #
          ):  #
               return True

          return False


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					List[str]						     =       []
          for char in text:
               __A  :					Dict						     =       ord(lowerCamelCase		)
               if cp == 0 or cp == 0xFFFD or _is_control(lowerCamelCase		):
                    continue
               if _is_whitespace(lowerCamelCase		):
                    output.append(" "		)
               else:
                    output.append(lowerCamelCase		)
          return "".join(lowerCamelCase		)


def 		_lowercase   (SCREAMING_SNAKE_CASE       ):




     '''simple docstring'''





     __A  :					Dict						     =       set()
     __A  :					Tuple						     =       word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)       )
          __A  :					str						     =       char
     return pairs


def 		_lowercase   (SCREAMING_SNAKE_CASE       ):




     '''simple docstring'''





     __A  :					Dict						     =       text.replace("—"					,				"-"       )
     __A  :					Optional[int]						     =       text.replace("–"					,				"-"       )
     __A  :					Dict						     =       text.replace("―"					,				"-"       )
     __A  :					Optional[int]						     =       text.replace("…"					,				"..."       )
     __A  :					int						     =       text.replace("´"					,				"'"       )
     __A  :					List[Any]						     =       re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"					,				r" \1 "					,				SCREAMING_SNAKE_CASE       )
     __A  :					Union[str, Any]						     =       re.sub(r"\s*\n\s*"					,				" \n "					,				SCREAMING_SNAKE_CASE       )
     __A  :					List[Any]						     =       re.sub(r"[^\S\n]+"					,				" "					,				SCREAMING_SNAKE_CASE       )
     return text.strip()







class        __magic_name__  (					lowerCAmelCase   ):

     """simple docstring"""






     lowerCamelCase__						=							VOCAB_FILES_NAMES
     lowerCamelCase__						=							PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase__						=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase__						=							['input_ids', 'attention_mask']


     def __init__(      self ,    lowerCamelCase ,    lowerCamelCase ,    lowerCamelCase="<unk>" ,    **lowerCamelCase		):


          '''simple docstring'''





          super().__init__(unk_token=lowerCamelCase ,    **lowerCamelCase		)

          try:
               import ftfy
               from spacy.lang.en import English

               __A  :					Union[str, Any]						     =       English()
               __A  :					Tuple						     =       _nlp.tokenizer
               __A  :					Any						     =       ftfy.fix_text
          except ImportError:
               logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."		)
               __A  :					List[Any]						     =       BasicTokenizer(do_lower_case=lowerCamelCase		)
               __A  :					Any						     =       None

          with open(lowerCamelCase ,    encoding="utf-8"		) as vocab_handle:
               __A  :					int						     =       json.load(lowerCamelCase		)
          __A  :					Union[str, Any]						     =       {v: k for k, v in self.encoder.items()}
          with open(lowerCamelCase ,    encoding="utf-8"		) as merges_handle:
               __A  :					Tuple						     =       merges_handle.read().split("\n"		)[1:-1]
          __A  :					List[Any]						     =       [tuple(merge.split()		) for merge in merges]
          __A  :					Optional[int]						     =       dict(zip(lowerCamelCase ,    range(len(lowerCamelCase		)		)		)		)
          __A  :					Union[str, Any]						     =       {}


     @property
     def lowerCAmelCase__     (      self		):


          '''simple docstring'''





          return True


     @property
     def lowerCAmelCase__     (      self		):


          '''simple docstring'''





          return len(self.encoder		)


     def lowerCAmelCase__     (      self		):


          '''simple docstring'''





          return dict(self.encoder ,    **self.added_tokens_encoder		)


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					Any						     =       tuple(token[:-1]		) + (token[-1] + "</w>",)
          if token in self.cache:
               return self.cache[token]
          __A  :					Optional[Any]						     =       get_pairs(lowerCamelCase		)

          if not pairs:
               return token + "</w>"

          while True:
               __A  :					List[Any]						     =       min(lowerCamelCase ,    key=lambda lowerCamelCase		: self.bpe_ranks.get(lowerCamelCase ,    float("inf"		)		)		)
               if bigram not in self.bpe_ranks:
                    break
               __A     ,__A  :					Tuple						     =       bigram
               __A  :					List[Any]						     =       []
               __A  :					Optional[int]						     =       0
               while i < len(lowerCamelCase		):
                    try:
                         __A  :					Dict						     =       word.index(lowerCamelCase ,    lowerCamelCase		)
                    except ValueError:
                         new_word.extend(word[i:]		)
                         break
                    else:
                         new_word.extend(word[i:j]		)
                         __A  :					int						     =       j

                    if word[i] == first and i < len(lowerCamelCase		) - 1 and word[i + 1] == second:
                         new_word.append(first + second		)
                         i += 2
                    else:
                         new_word.append(word[i]		)
                         i += 1
               __A  :					List[str]						     =       tuple(lowerCamelCase		)
               __A  :					int						     =       new_word
               if len(lowerCamelCase		) == 1:
                    break
               else:
                    __A  :					List[str]						     =       get_pairs(lowerCamelCase		)
          __A  :					Tuple						     =       " ".join(lowerCamelCase		)
          if word == "\n  </w>":
               __A  :					List[str]						     =       "\n</w>"
          __A  :					List[Any]						     =       word
          return word


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					Dict						     =       []
          if self.fix_text is None:
               # Using BERT's BasicTokenizer
               __A  :					Tuple						     =       self.nlp.tokenize(lowerCamelCase		)
               for token in text:
                    split_tokens.extend(list(self.bpe(lowerCamelCase		).split(" "		)		)		)
          else:
               # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
               __A  :					List[Any]						     =       self.nlp(text_standardize(self.fix_text(lowerCamelCase		)		)		)
               for token in text:
                    split_tokens.extend(list(self.bpe(token.text.lower()		).split(" "		)		)		)
          return split_tokens


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          return self.encoder.get(lowerCamelCase ,    self.encoder.get(self.unk_token		)		)


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          return self.decoder.get(lowerCamelCase ,    self.unk_token		)


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					Any						     =       "".join(lowerCamelCase		).replace("</w>" ,    " "		).strip()
          return out_string


     def lowerCAmelCase__     (      self ,    lowerCamelCase ,    lowerCamelCase = None		):


          '''simple docstring'''





          if not os.path.isdir(lowerCamelCase		):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
               return
          __A  :					Optional[Any]						     =       os.path.join(
              lowerCamelCase ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
          __A  :					str						     =       os.path.join(
              lowerCamelCase ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

          with open(lowerCamelCase ,    "w" ,    encoding="utf-8"		) as f:
               f.write(json.dumps(self.encoder ,    indent=2 ,    sort_keys=lowerCamelCase ,    ensure_ascii=lowerCamelCase		) + "\n"		)

          __A  :					Tuple						     =       0
          with open(lowerCamelCase ,    "w" ,    encoding="utf-8"		) as writer:
               writer.write("#version: 0.2\n"		)
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,    key=lambda lowerCamelCase		: kv[1]		):
                    if index != token_index:
                         logger.warning(
                             f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                             " Please check that the tokenizer is not corrupted!"		)
                         __A  :					Tuple						     =       token_index
                    writer.write(" ".join(lowerCamelCase		) + "\n"		)
                    index += 1

          return vocab_file, merge_file

