

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowerCAmelCase						:   List[Any]										=    logging.get_logger(__name__)

lowerCAmelCase						:   List[str]										=    {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

lowerCAmelCase						:   List[str]										=    {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

lowerCAmelCase						:   Any										=    {
    '''openai-gpt''': 512,
}





def 			_lowercase    (      __UpperCamelCase     :		Union[str, Any] ):
      snake_case__					=   text.strip()
      if not text:
            return []
      snake_case__					=   text.split()
      return tokens
class     SCREAMING_SNAKE_CASE__     (						__a		):





       '''simple docstring'''




       def __init__(						self       :						Dict    ,		lowerCAmelCase__       :						Optional[int]=True    ,		lowerCAmelCase__       :						Dict=None    ,		lowerCAmelCase__       :						Any=True    ,		lowerCAmelCase__       :						List[str]=None    ,		lowerCAmelCase__       :						int=True    ,		)			->    Any:
             if never_split is None:
                   snake_case__					=   []
             snake_case__					=   do_lower_case
             snake_case__					=   set(lowerCAmelCase__							)
             snake_case__					=   tokenize_chinese_chars
             snake_case__					=   strip_accents
             snake_case__					=   do_split_on_punc

       def UpperCAmelCase_						(						self       :						Optional[Any]    ,		lowerCAmelCase__       :						str    ,		lowerCAmelCase__       :						Tuple=None							)			->    List[Any]:
             snake_case__					=   self.never_split.union(set(lowerCAmelCase__							)							) if never_split else self.never_split
             snake_case__					=   self._clean_text(lowerCAmelCase__							)

             # This was added on November 1st, 2018 for the multilingual and Chinese
             # models. This is also applied to the English models now, but it doesn't
             # matter since the English models were not trained on any Chinese data
             # and generally don't have any Chinese data in them (there are Chinese
             # characters in the vocabulary because Wikipedia does have some Chinese
             # words in the English Wikipedia.).
             if self.tokenize_chinese_chars:
                   snake_case__					=   self._tokenize_chinese_chars(lowerCAmelCase__							)
             # prevents treating the same character with different unicode codepoints as different characters
             snake_case__					=   unicodedata.normalize("""NFC"""    ,		lowerCAmelCase__							)
             snake_case__					=   whitespace_tokenize(lowerCAmelCase__							)
             snake_case__					=   []
             for token in orig_tokens:
                   if token not in never_split:
                         if self.do_lower_case:
                               snake_case__					=   token.lower()
                               if self.strip_accents is not False:
                                     snake_case__					=   self._run_strip_accents(lowerCAmelCase__							)
                         elif self.strip_accents:
                               snake_case__					=   self._run_strip_accents(lowerCAmelCase__							)
                   split_tokens.extend(self._run_split_on_punc(lowerCAmelCase__    ,		lowerCAmelCase__							)							)

             snake_case__					=   whitespace_tokenize(""" """.join(lowerCAmelCase__							)							)
             return output_tokens

       def UpperCAmelCase_						(						self       :						Dict    ,		lowerCAmelCase__       :						int							)			->    Optional[Any]:
             snake_case__					=   unicodedata.normalize("""NFD"""    ,		lowerCAmelCase__							)
             snake_case__					=   []
             for char in text:
                   snake_case__					=   unicodedata.category(lowerCAmelCase__							)
                   if cat == "Mn":
                         continue
                   output.append(lowerCAmelCase__							)
             return "".join(lowerCAmelCase__							)

       def UpperCAmelCase_						(						self       :						Any    ,		lowerCAmelCase__       :						List[str]    ,		lowerCAmelCase__       :						Dict=None							)			->    List[str]:
             if not self.do_split_on_punc or (never_split is not None and text in never_split):
                   return [text]
             snake_case__					=   list(lowerCAmelCase__							)
             snake_case__					=   0
             snake_case__					=   True
             snake_case__					=   []
             while i < len(lowerCAmelCase__							):
                   snake_case__					=   chars[i]
                   if _is_punctuation(lowerCAmelCase__							):
                         output.append([char]							)
                         snake_case__					=   True
                   else:
                         if start_new_word:
                               output.append([]							)
                         snake_case__					=   False
                         output[-1].append(lowerCAmelCase__							)
                   i += 1

             return ["".join(lowerCAmelCase__							) for x in output]

       def UpperCAmelCase_						(						self       :						Union[str, Any]    ,		lowerCAmelCase__       :						str							)			->    Optional[int]:
             snake_case__					=   []
             for char in text:
                   snake_case__					=   ord(lowerCAmelCase__							)
                   if self._is_chinese_char(lowerCAmelCase__							):
                         output.append(""" """							)
                         output.append(lowerCAmelCase__							)
                         output.append(""" """							)
                   else:
                         output.append(lowerCAmelCase__							)
             return "".join(lowerCAmelCase__							)

       def UpperCAmelCase_						(						self       :						Union[str, Any]    ,		lowerCAmelCase__       :						Union[str, Any]							)			->    Optional[Any]:
             # This defines a "chinese character" as anything in the CJK Unicode block:
             #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
             #
             # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
             # despite its name. The modern Korean Hangul alphabet is a different block,
             # as is Japanese Hiragana and Katakana. Those alphabets are used to write
             # space-separated words, so they are not treated specially and handled
             # like the all of the other languages.
             if (
                 (cp >= 0X4_E_0_0 and cp <= 0X9_F_F_F)
                 or (cp >= 0X3_4_0_0 and cp <= 0X4_D_B_F)  #
                 or (cp >= 0X2_0_0_0_0 and cp <= 0X2_A_6_D_F)  #
                 or (cp >= 0X2_A_7_0_0 and cp <= 0X2_B_7_3_F)  #
                 or (cp >= 0X2_B_7_4_0 and cp <= 0X2_B_8_1_F)  #
                 or (cp >= 0X2_B_8_2_0 and cp <= 0X2_C_E_A_F)  #
                 or (cp >= 0XF_9_0_0 and cp <= 0XF_A_F_F)
                 or (cp >= 0X2_F_8_0_0 and cp <= 0X2_F_A_1_F)  #
             ):  #
                   return True

             return False

       def UpperCAmelCase_						(						self       :						Dict    ,		lowerCAmelCase__       :						Tuple							)			->    int:
             snake_case__					=   []
             for char in text:
                   snake_case__					=   ord(lowerCAmelCase__							)
                   if cp == 0 or cp == 0XF_F_F_D or _is_control(lowerCAmelCase__							):
                         continue
                   if _is_whitespace(lowerCAmelCase__							):
                         output.append(""" """							)
                   else:
                         output.append(lowerCAmelCase__							)
             return "".join(lowerCAmelCase__							)





def 			_lowercase    (      __UpperCamelCase     :		Tuple ):
      snake_case__					=   set()
      snake_case__					=   word[0]
      for char in word[1:]:
            pairs.add((prev_char, char) )
            snake_case__					=   char
      return pairs





def 			_lowercase    (      __UpperCamelCase     :		int ):
      snake_case__					=   text.replace("""—"""   ,       """-""" )
      snake_case__					=   text.replace("""–"""   ,       """-""" )
      snake_case__					=   text.replace("""―"""   ,       """-""" )
      snake_case__					=   text.replace("""…"""   ,       """...""" )
      snake_case__					=   text.replace("""´"""   ,       """'""" )
      snake_case__					=   re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""   ,       R""" \1 """   ,       __UpperCamelCase )
      snake_case__					=   re.sub(R"""\s*\n\s*"""   ,       """ \n """   ,       __UpperCamelCase )
      snake_case__					=   re.sub(R"""[^\S\n]+"""   ,       """ """   ,       __UpperCamelCase )
      return text.strip()
class     SCREAMING_SNAKE_CASE__     (						__a		):





       '''simple docstring'''




       UpperCamelCase__ :     List[str]				    =       VOCAB_FILES_NAMES
       UpperCamelCase__ :     Union[str, Any]				    =       PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase__ :     Union[str, Any]				    =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase__ :     Optional[int]				    =       ['''input_ids''', '''attention_mask''']

       def __init__(						self       :						Any    ,		lowerCAmelCase__       :						Optional[int]    ,		lowerCAmelCase__       :						List[Any]    ,		lowerCAmelCase__       :						List[Any]="<unk>"    ,		**lowerCAmelCase__       :						int							)			->    str:
             super().__init__(unk_token=lowerCAmelCase__    ,		**lowerCAmelCase__							)

             try:
                   import ftfy
                   from spacy.lang.en import English

                   snake_case__					=   English()
                   snake_case__					=   _nlp.tokenizer
                   snake_case__					=   ftfy.fix_text
             except ImportError:
                   logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""							)
                   snake_case__					=   BasicTokenizer(do_lower_case=lowerCAmelCase__							)
                   snake_case__					=   None

             with open(lowerCAmelCase__    ,		encoding="""utf-8"""							) as vocab_handle:
                   snake_case__					=   json.load(lowerCAmelCase__							)
             snake_case__					=   {v: k for k, v in self.encoder.items()}
             with open(lowerCAmelCase__    ,		encoding="""utf-8"""							) as merges_handle:
                   snake_case__					=   merges_handle.read().split("""\n"""							)[1:-1]
             snake_case__					=   [tuple(merge.split()							) for merge in merges]
             snake_case__					=   dict(zip(lowerCAmelCase__    ,		range(len(lowerCAmelCase__							)							)							)							)
             snake_case__					=   {}

       @property
       def UpperCAmelCase_						(						self       :						Tuple							)			->    Tuple:
             return True

       @property
       def UpperCAmelCase_						(						self       :						Union[str, Any]							)			->    Tuple:
             return len(self.encoder							)

       def UpperCAmelCase_						(						self       :						List[Any]							)			->    Tuple:
             return dict(self.encoder    ,		**self.added_tokens_encoder							)

       def UpperCAmelCase_						(						self       :						Dict    ,		lowerCAmelCase__       :						Optional[int]							)			->    Any:
             snake_case__					=   tuple(token[:-1]							) + (token[-1] + """</w>""",)
             if token in self.cache:
                   return self.cache[token]
             snake_case__					=   get_pairs(lowerCAmelCase__							)

             if not pairs:
                   return token + "</w>"

             while True:
                   snake_case__					=   min(lowerCAmelCase__    ,		key=lambda lowerCAmelCase__							: self.bpe_ranks.get(lowerCAmelCase__    ,		float("""inf"""							)							)							)
                   if bigram not in self.bpe_ranks:
                         break
                   snake_case__	,			snake_case__					=   bigram
                   snake_case__					=   []
                   snake_case__					=   0
                   while i < len(lowerCAmelCase__							):
                         try:
                               snake_case__					=   word.index(lowerCAmelCase__    ,		lowerCAmelCase__							)
                         except ValueError:
                               new_word.extend(word[i:]							)
                               break
                         else:
                               new_word.extend(word[i:j]							)
                               snake_case__					=   j

                         if word[i] == first and i < len(lowerCAmelCase__							) - 1 and word[i + 1] == second:
                               new_word.append(first + second							)
                               i += 2
                         else:
                               new_word.append(word[i]							)
                               i += 1
                   snake_case__					=   tuple(lowerCAmelCase__							)
                   snake_case__					=   new_word
                   if len(lowerCAmelCase__							) == 1:
                         break
                   else:
                         snake_case__					=   get_pairs(lowerCAmelCase__							)
             snake_case__					=   """ """.join(lowerCAmelCase__							)
             if word == "\n  </w>":
                   snake_case__					=   """\n</w>"""
             snake_case__					=   word
             return word

       def UpperCAmelCase_						(						self       :						Optional[int]    ,		lowerCAmelCase__       :						Any							)			->    Any:
             snake_case__					=   []
             if self.fix_text is None:
                   # Using BERT's BasicTokenizer
                   snake_case__					=   self.nlp.tokenize(lowerCAmelCase__							)
                   for token in text:
                         split_tokens.extend(list(self.bpe(lowerCAmelCase__							).split(""" """							)							)							)
             else:
                   # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
                   snake_case__					=   self.nlp(text_standardize(self.fix_text(lowerCAmelCase__							)							)							)
                   for token in text:
                         split_tokens.extend(list(self.bpe(token.text.lower()							).split(""" """							)							)							)
             return split_tokens

       def UpperCAmelCase_						(						self       :						str    ,		lowerCAmelCase__       :						Optional[int]							)			->    int:
             return self.encoder.get(lowerCAmelCase__    ,		self.encoder.get(self.unk_token							)							)

       def UpperCAmelCase_						(						self       :						Dict    ,		lowerCAmelCase__       :						Dict							)			->    Tuple:
             return self.decoder.get(lowerCAmelCase__    ,		self.unk_token							)

       def UpperCAmelCase_						(						self       :						int    ,		lowerCAmelCase__       :						Dict							)			->    List[Any]:
             snake_case__					=   """""".join(lowerCAmelCase__							).replace("""</w>"""    ,		""" """							).strip()
             return out_string

       def UpperCAmelCase_						(						self       :						Tuple    ,		lowerCAmelCase__       :						str    ,		lowerCAmelCase__       :						Optional[str] = None							)			->    Tuple[str]:
             if not os.path.isdir(lowerCAmelCase__							):
                   logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''							)
                   return
             snake_case__					=   os.path.join(
                 lowerCAmelCase__    ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
             snake_case__					=   os.path.join(
                 lowerCAmelCase__    ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]							)

             with open(lowerCAmelCase__    ,		"""w"""    ,		encoding="""utf-8"""							) as f:
                   f.write(json.dumps(self.encoder    ,		indent=2    ,		sort_keys=lowerCAmelCase__    ,		ensure_ascii=lowerCAmelCase__							) + """\n"""							)

             snake_case__					=   0
             with open(lowerCAmelCase__    ,		"""w"""    ,		encoding="""utf-8"""							) as writer:
                   writer.write("""#version: 0.2\n"""							)
                   for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,		key=lambda lowerCAmelCase__							: kv[1]							):
                         if index != token_index:
                               logger.warning(
                                   f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                   """ Please check that the tokenizer is not corrupted!"""							)
                               snake_case__					=   token_index
                         writer.write(""" """.join(lowerCAmelCase__							) + """\n"""							)
                         index += 1

             return vocab_file, merge_file









from doctest import testmod
from math import sqrt





def 			_lowercase    (      __UpperCamelCase     :		int ):
      snake_case__					=   []
      if num < 1:
            return facs
      facs.append(1 )
      if num == 1:
            return facs
      facs.append(__UpperCamelCase )
      for i in range(2   ,       int(sqrt(__UpperCamelCase ) ) + 1 ):
            if num % i == 0:  # If i is a factor of num
                  facs.append(__UpperCamelCase )
                  snake_case__					=   num // i  # num//i is the other factor of num
                  if d != i:  # If d and i are distinct
                        facs.append(__UpperCamelCase )  # we have found another factor
      facs.sort()
      return facs


if __name__ == "__main__":
       testmod(name='''factors_of_a_number''', verbose=True)






