

'''simple docstring'''





import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCamelCase__		   =    logging.get_logger(__name__)

UpperCamelCase__		   =    {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

UpperCamelCase__		   =    {
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

UpperCamelCase__		   =    {
    'openai-gpt': 512,
}







def 	lowerCamelCase__			(   __A						:Any		):


       """simple docstring"""




       __snake_case					  =       text.strip()
       if not text:
              return []
       __snake_case					  =       text.split()
       return tokens

class 						__snake_case						(							snake_case__     ):
       """simple docstring"""







       def __init__(		self			,   _UpperCamelCase=True			,   _UpperCamelCase=None			,   _UpperCamelCase=True			,   _UpperCamelCase=None			,   _UpperCamelCase=True			,   )			->       Optional[Any]:






              """simple docstring"""


              if never_split is None:
                     __snake_case					  =       []
              __snake_case					  =       do_lower_case
              __snake_case					  =       set(A_      )
              __snake_case					  =       tokenize_chinese_chars
              __snake_case					  =       strip_accents
              __snake_case					  =       do_split_on_punc


       def  a		(		self			,   _UpperCamelCase			,   _UpperCamelCase=None      )			->       Tuple:






              """simple docstring"""


              __snake_case					  =       self.never_split.union(set(A_      )      ) if never_split else self.never_split
              __snake_case					  =       self._clean_text(A_      )

              # This was added on November 1st, 2018 for the multilingual and Chinese
              # models. This is also applied to the English models now, but it doesn't
              # matter since the English models were not trained on any Chinese data
              # and generally don't have any Chinese data in them (there are Chinese
              # characters in the vocabulary because Wikipedia does have some Chinese
              # words in the English Wikipedia.).
              if self.tokenize_chinese_chars:
                     __snake_case					  =       self._tokenize_chinese_chars(A_      )
              # prevents treating the same character with different unicode codepoints as different characters
              __snake_case					  =       unicodedata.normalize("""NFC"""			,   A_      )
              __snake_case					  =       whitespace_tokenize(A_      )
              __snake_case					  =       []
              for token in orig_tokens:
                     if token not in never_split:
                            if self.do_lower_case:
                                   __snake_case					  =       token.lower()
                                   if self.strip_accents is not False:
                                          __snake_case					  =       self._run_strip_accents(A_      )
                            elif self.strip_accents:
                                   __snake_case					  =       self._run_strip_accents(A_      )
                     split_tokens.extend(self._run_split_on_punc(A_			,   A_      )      )

              __snake_case					  =       whitespace_tokenize(""" """.join(A_      )      )
              return output_tokens


       def  a		(		self			,   _UpperCamelCase      )			->       Dict:






              """simple docstring"""


              __snake_case					  =       unicodedata.normalize("""NFD"""			,   A_      )
              __snake_case					  =       []
              for char in text:
                     __snake_case					  =       unicodedata.category(A_      )
                     if cat == "Mn":
                            continue
                     output.append(A_      )
              return "".join(A_      )


       def  a		(		self			,   _UpperCamelCase			,   _UpperCamelCase=None      )			->       str:






              """simple docstring"""


              if not self.do_split_on_punc or (never_split is not None and text in never_split):
                     return [text]
              __snake_case					  =       list(A_      )
              __snake_case					  =       0
              __snake_case					  =       True
              __snake_case					  =       []
              while i < len(A_      ):
                     __snake_case					  =       chars[i]
                     if _is_punctuation(A_      ):
                            output.append([char]      )
                            __snake_case					  =       True
                     else:
                            if start_new_word:
                                   output.append([]      )
                            __snake_case					  =       False
                            output[-1].append(A_      )
                     i += 1

              return ["".join(A_      ) for x in output]


       def  a		(		self			,   _UpperCamelCase      )			->       Optional[Any]:






              """simple docstring"""


              __snake_case					  =       []
              for char in text:
                     __snake_case					  =       ord(A_      )
                     if self._is_chinese_char(A_      ):
                            output.append(""" """      )
                            output.append(A_      )
                            output.append(""" """      )
                     else:
                            output.append(A_      )
              return "".join(A_      )


       def  a		(		self			,   _UpperCamelCase      )			->       Tuple:






              """simple docstring"""


              if (
                  (cp >= 0x4E_00 and cp <= 0x9F_FF)
                  or (cp >= 0x34_00 and cp <= 0x4D_BF)  #
                  or (cp >= 0x2_00_00 and cp <= 0x2_A6_DF)  #
                  or (cp >= 0x2_A7_00 and cp <= 0x2_B7_3F)  #
                  or (cp >= 0x2_B7_40 and cp <= 0x2_B8_1F)  #
                  or (cp >= 0x2_B8_20 and cp <= 0x2_CE_AF)  #
                  or (cp >= 0xF9_00 and cp <= 0xFA_FF)
                  or (cp >= 0x2_F8_00 and cp <= 0x2_FA_1F)  #
              ):  #
                     return True

              return False






       def  a		(		self			,   _UpperCamelCase      )			->       Tuple:






              """simple docstring"""


              __snake_case					  =       []
              for char in text:
                     __snake_case					  =       ord(A_      )
                     if cp == 0 or cp == 0xFF_FD or _is_control(A_      ):
                            continue
                     if _is_whitespace(A_      ):
                            output.append(""" """      )
                     else:
                            output.append(A_      )
              return "".join(A_      )







def 	lowerCamelCase__			(   __A						:List[str]		):


       """simple docstring"""




       __snake_case					  =       set()
       __snake_case					  =       word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)		)
              __snake_case					  =       char
       return pairs







def 	lowerCamelCase__			(   __A						:Optional[Any]		):


       """simple docstring"""




       __snake_case					  =       text.replace("""—"""  ,"""-"""		)
       __snake_case					  =       text.replace("""–"""  ,"""-"""		)
       __snake_case					  =       text.replace("""―"""  ,"""-"""		)
       __snake_case					  =       text.replace("""…"""  ,"""..."""		)
       __snake_case					  =       text.replace("""´"""  ,"""'"""		)
       __snake_case					  =       re.sub(r"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""  ,r""" \1 """  ,lowercase__		)
       __snake_case					  =       re.sub(r"""\s*\n\s*"""  ,""" \n """  ,lowercase__		)
       __snake_case					  =       re.sub(r"""[^\S\n]+"""  ,""" """  ,lowercase__		)
       return text.strip()




class 						__snake_case						(							snake_case__     ):
       """simple docstring"""







       __SCREAMING_SNAKE_CASE													=       VOCAB_FILES_NAMES
       __SCREAMING_SNAKE_CASE													=       PRETRAINED_VOCAB_FILES_MAP
       __SCREAMING_SNAKE_CASE													=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __SCREAMING_SNAKE_CASE													=       ["input_ids", "attention_mask"]


       def __init__(		self			,   _UpperCamelCase			,   _UpperCamelCase			,   _UpperCamelCase="<unk>"			,   **_UpperCamelCase      )			->       Any:






              """simple docstring"""


              super().__init__(unk_token=A_			,   **A_      )

              try:
                     import ftfy
                     from spacy.lang.en import English

                     __snake_case					  =       English()
                     __snake_case					  =       _nlp.tokenizer
                     __snake_case					  =       ftfy.fix_text
              except ImportError:
                     logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""      )
                     __snake_case					  =       BasicTokenizer(do_lower_case=A_      )
                     __snake_case					  =       None

              with open(A_			,   encoding="""utf-8"""      ) as vocab_handle:
                     __snake_case					  =       json.load(A_      )
              __snake_case					  =       {v: k for k, v in self.encoder.items()}
              with open(A_			,   encoding="""utf-8"""      ) as merges_handle:
                     __snake_case					  =       merges_handle.read().split("""\n"""      )[1:-1]
              __snake_case					  =       [tuple(merge.split()      ) for merge in merges]
              __snake_case					  =       dict(zip(A_			,   range(len(A_      )      )      )      )
              __snake_case					  =       {}


       @property
       def  a		(		self      )			->       int:






              """simple docstring"""


              return True


       @property
       def  a		(		self      )			->       int:






              """simple docstring"""


              return len(self.encoder      )


       def  a		(		self      )			->       Union[str, Any]:






              """simple docstring"""


              return dict(self.encoder			,   **self.added_tokens_encoder      )


       def  a		(		self			,   _UpperCamelCase      )			->       str:






              """simple docstring"""


              __snake_case					  =       tuple(token[:-1]      ) + (token[-1] + """</w>""",)
              if token in self.cache:
                     return self.cache[token]
              __snake_case					  =       get_pairs(A_      )

              if not pairs:
                     return token + "</w>"

              while True:
                     __snake_case					  =       min(A_			,   key=lambda _UpperCamelCase      : self.bpe_ranks.get(A_			,   float("""inf"""      )      )      )
                     if bigram not in self.bpe_ranks:
                            break
                     __snake_case  ,						__snake_case					  =       bigram
                     __snake_case					  =       []
                     __snake_case					  =       0
                     while i < len(A_      ):
                            try:
                                   __snake_case					  =       word.index(A_			,   A_      )
                            except ValueError:
                                   new_word.extend(word[i:]      )
                                   break
                            else:
                                   new_word.extend(word[i:j]      )
                                   __snake_case					  =       j

                            if word[i] == first and i < len(A_      ) - 1 and word[i + 1] == second:
                                   new_word.append(first + second      )
                                   i += 2
                            else:
                                   new_word.append(word[i]      )
                                   i += 1
                     __snake_case					  =       tuple(A_      )
                     __snake_case					  =       new_word
                     if len(A_      ) == 1:
                            break
                     else:
                            __snake_case					  =       get_pairs(A_      )
              __snake_case					  =       """ """.join(A_      )
              if word == "\n  </w>":
                     __snake_case					  =       """\n</w>"""
              __snake_case					  =       word
              return word


       def  a		(		self			,   _UpperCamelCase      )			->       Optional[int]:






              """simple docstring"""


              __snake_case					  =       []
              if self.fix_text is None:
                     # Using BERT's BasicTokenizer
                     __snake_case					  =       self.nlp.tokenize(A_      )
                     for token in text:
                            split_tokens.extend(list(self.bpe(A_      ).split(""" """      )      )      )
              else:
                     # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
                     __snake_case					  =       self.nlp(text_standardize(self.fix_text(A_      )      )      )
                     for token in text:
                            split_tokens.extend(list(self.bpe(token.text.lower()      ).split(""" """      )      )      )
              return split_tokens


       def  a		(		self			,   _UpperCamelCase      )			->       Optional[int]:






              """simple docstring"""


              return self.encoder.get(A_			,   self.encoder.get(self.unk_token      )      )


       def  a		(		self			,   _UpperCamelCase      )			->       str:






              """simple docstring"""


              return self.decoder.get(A_			,   self.unk_token      )


       def  a		(		self			,   _UpperCamelCase      )			->       Union[str, Any]:






              """simple docstring"""


              __snake_case					  =       """""".join(A_      ).replace("""</w>"""			,   """ """      ).strip()
              return out_string






       def  a		(		self			,   _UpperCamelCase			,   _UpperCamelCase = None      )			->       Optional[int]:






              """simple docstring"""


              if not os.path.isdir(A_      ):
                     logger.error(F'Vocabulary path ({save_directory}) should be a directory'      )
                     return
              __snake_case					  =       os.path.join(
                  A_			,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )
              __snake_case					  =       os.path.join(
                  A_			,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]      )

              with open(A_			,   """w"""			,   encoding="""utf-8"""      ) as f:
                     f.write(json.dumps(self.encoder			,   indent=2			,   sort_keys=A_			,   ensure_ascii=A_      ) + """\n"""      )

              __snake_case					  =       0
              with open(A_			,   """w"""			,   encoding="""utf-8"""      ) as writer:
                     writer.write("""#version: 0.2\n"""      )
                     for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,   key=lambda _UpperCamelCase      : kv[1]      ):
                            if index != token_index:
                                   logger.warning(
                                       F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                                       """ Please check that the tokenizer is not corrupted!"""      )
                                   __snake_case					  =       token_index
                            writer.write(""" """.join(A_      ) + """\n"""      )
                            index += 1

              return vocab_file, merge_file




from __future__ import annotations

class 						__snake_case						:
       """simple docstring"""







       def __init__(		self			,   _UpperCamelCase      )			->       None:






              """simple docstring"""


              __snake_case					  =       value
              __snake_case					  =       None
              __snake_case					  =       None






class 						__snake_case						:
       """simple docstring"""







       __SCREAMING_SNAKE_CASE													=       42


       def __init__(		self      )			->       None:






              """simple docstring"""


              __snake_case					  =       0


       def  a		(		self			,   _UpperCamelCase			,   _UpperCamelCase      )			->       None:






              """simple docstring"""


              if node is None:
                     return

              if path_sum == self.target:
                     self.paths += 1

              if node.left:
                     self.depth_first_search(node.left			,   path_sum + node.left.value      )
              if node.right:
                     self.depth_first_search(node.right			,   path_sum + node.right.value      )






       def  a		(		self			,   _UpperCamelCase			,   _UpperCamelCase = None      )			->       int:






              """simple docstring"""


              if node is None:
                     return 0
              if target is not None:
                     __snake_case					  =       target

              self.depth_first_search(_UpperCamelCase			,   node.value      )
              self.path_sum(node.left      )
              self.path_sum(node.right      )

              return self.paths


if __name__ == "__main__":
    import doctest

    doctest.testmod()

