




'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a_   :	Optional[int]							   = logging.get_logger(__name__)

a_   :	Union[str, Any]							   = {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

a_   :	List[Any]							   = {
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
}

a_   :	Optional[int]							   = {
    """roberta-base""": 5_12,
    """roberta-large""": 5_12,
    """roberta-large-mnli""": 5_12,
    """distilroberta-base""": 5_12,
    """roberta-base-openai-detector""": 5_12,
    """roberta-large-openai-detector""": 5_12,
}





@lru_cache()
def   a_							(   )							->		Optional[int]:



       """simple docstring"""
       lowerCamelCase_							=(
           list(range(ord('''!'''     )     , ord('''~'''     ) + 1     )     ) + list(range(ord('''¡'''     )     , ord('''¬'''     ) + 1     )     ) + list(range(ord('''®'''     )     , ord('''ÿ'''     ) + 1     )     )
       )
       lowerCamelCase_							=bs[:]
       lowerCamelCase_							=0
       for b in range(2**8     ):
              if b not in bs:
                     bs.append(__snake_case     )
                     cs.append(2**8 + n     )
                     n += 1
       lowerCamelCase_							=[chr(__snake_case     ) for n in cs]
       return dict(zip(__snake_case     , __snake_case     )     )





def   a_							(   __snake_case							:    str     )							->		Optional[int]:



       """simple docstring"""
       lowerCamelCase_							=set()
       lowerCamelCase_							=word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)     )
              lowerCamelCase_							=char
       return pairs


class 						__UpperCamelCase (				lowerCamelCase__       ):
       lowercase			:					Union[str, Any]   		=VOCAB_FILES_NAMES
       lowercase			:					Any   		=PRETRAINED_VOCAB_FILES_MAP
       lowercase			:					List[str]   		=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowercase			:					Dict   		=['input_ids', 'attention_mask']






       def __init__(							self,	lowerCAmelCase,	lowerCAmelCase,	lowerCAmelCase="replace",	lowerCAmelCase="<s>",	lowerCAmelCase="</s>",	lowerCAmelCase="</s>",	lowerCAmelCase="<s>",	lowerCAmelCase="<unk>",	lowerCAmelCase="<pad>",	lowerCAmelCase="<mask>",	lowerCAmelCase=False,	**lowerCAmelCase,	):



              """simple docstring"""







              lowerCamelCase_							=AddedToken(lowerCAmelCase,	lstrip=lowerCAmelCase,	rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase,	lowerCAmelCase						) else bos_token
              lowerCamelCase_							=AddedToken(lowerCAmelCase,	lstrip=lowerCAmelCase,	rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase,	lowerCAmelCase						) else eos_token
              lowerCamelCase_							=AddedToken(lowerCAmelCase,	lstrip=lowerCAmelCase,	rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase,	lowerCAmelCase						) else sep_token
              lowerCamelCase_							=AddedToken(lowerCAmelCase,	lstrip=lowerCAmelCase,	rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase,	lowerCAmelCase						) else cls_token
              lowerCamelCase_							=AddedToken(lowerCAmelCase,	lstrip=lowerCAmelCase,	rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase,	lowerCAmelCase						) else unk_token
              lowerCamelCase_							=AddedToken(lowerCAmelCase,	lstrip=lowerCAmelCase,	rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase,	lowerCAmelCase						) else pad_token

              # Mask token behave like a normal word, i.e. include the space before it
              lowerCamelCase_							=AddedToken(lowerCAmelCase,	lstrip=lowerCAmelCase,	rstrip=lowerCAmelCase						) if isinstance(lowerCAmelCase,	lowerCAmelCase						) else mask_token

              super().__init__(
                  errors=lowerCAmelCase,	bos_token=lowerCAmelCase,	eos_token=lowerCAmelCase,	unk_token=lowerCAmelCase,	sep_token=lowerCAmelCase,	cls_token=lowerCAmelCase,	pad_token=lowerCAmelCase,	mask_token=lowerCAmelCase,	add_prefix_space=lowerCAmelCase,	**lowerCAmelCase,	)

              with open(lowerCAmelCase,	encoding='''utf-8'''						) as vocab_handle:
                     lowerCamelCase_							=json.load(lowerCAmelCase						)
              lowerCamelCase_							={v: k for k, v in self.encoder.items()}
              lowerCamelCase_							=errors  # how to handle errors in decoding
              lowerCamelCase_							=bytes_to_unicode()
              lowerCamelCase_							={v: k for k, v in self.byte_encoder.items()}
              with open(lowerCAmelCase,	encoding='''utf-8'''						) as merges_handle:
                     lowerCamelCase_							=merges_handle.read().split('''\n'''						)[1:-1]
              lowerCamelCase_							=[tuple(merge.split()						) for merge in bpe_merges]
              lowerCamelCase_							=dict(zip(lowerCAmelCase,	range(len(lowerCAmelCase						)						)						)						)
              lowerCamelCase_							={}
              lowerCamelCase_							=add_prefix_space

              # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
              lowerCamelCase_							=re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''						)






       @property
       def   lowercase__  (							self						):



              """simple docstring"""







              return len(self.encoder						)






       def   lowercase__  (							self						):



              """simple docstring"""







              return dict(self.encoder,	**self.added_tokens_encoder						)






       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              if token in self.cache:
                     return self.cache[token]
              lowerCamelCase_							=tuple(lowerCAmelCase						)
              lowerCamelCase_							=get_pairs(lowerCAmelCase						)

              if not pairs:
                     return token

              while True:
                     lowerCamelCase_							=min(lowerCAmelCase,	key=lambda lowerCAmelCase						: self.bpe_ranks.get(lowerCAmelCase,	float('''inf'''						)						)						)
                     if bigram not in self.bpe_ranks:
                            break
                     lowerCamelCase_,   lowerCamelCase_							=bigram
                     lowerCamelCase_							=[]
                     lowerCamelCase_							=0
                     while i < len(lowerCAmelCase						):
                            try:
                                   lowerCamelCase_							=word.index(lowerCAmelCase,	lowerCAmelCase						)
                            except ValueError:
                                   new_word.extend(word[i:]						)
                                   break
                            else:
                                   new_word.extend(word[i:j]						)
                                   lowerCamelCase_							=j

                            if word[i] == first and i < len(lowerCAmelCase						) - 1 and word[i + 1] == second:
                                   new_word.append(first + second						)
                                   i += 2
                            else:
                                   new_word.append(word[i]						)
                                   i += 1
                     lowerCamelCase_							=tuple(lowerCAmelCase						)
                     lowerCamelCase_							=new_word
                     if len(lowerCAmelCase						) == 1:
                            break
                     else:
                            lowerCamelCase_							=get_pairs(lowerCAmelCase						)
              lowerCamelCase_							=''' '''.join(lowerCAmelCase						)
              lowerCamelCase_							=word
              return word






       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              lowerCamelCase_							=[]
              for token in re.findall(self.pat,	lowerCAmelCase						):
                     lowerCamelCase_							=''''''.join(
                         self.byte_encoder[b] for b in token.encode('''utf-8'''						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                     bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase						).split(''' '''						)						)
              return bpe_tokens






       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              return self.encoder.get(lowerCAmelCase,	self.encoder.get(self.unk_token						)						)






       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              return self.decoder.get(lowerCAmelCase						)






       def   lowercase__  (							self,	lowerCAmelCase						):



              """simple docstring"""







              lowerCamelCase_							=''''''.join(lowerCAmelCase						)
              lowerCamelCase_							=bytearray([self.byte_decoder[c] for c in text]						).decode('''utf-8''',	errors=self.errors						)
              return text






       def   lowercase__  (							self,	lowerCAmelCase,	lowerCAmelCase = None						):



              """simple docstring"""







              if not os.path.isdir(lowerCAmelCase						):
                     logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
                     return
              lowerCamelCase_							=os.path.join(
                  lowerCAmelCase,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
              lowerCamelCase_							=os.path.join(
                  lowerCAmelCase,	(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

              with open(lowerCAmelCase,	'''w''',	encoding='''utf-8'''						) as f:
                     f.write(json.dumps(self.encoder,	indent=2,	sort_keys=lowerCAmelCase,	ensure_ascii=lowerCAmelCase						) + '''\n'''						)

              lowerCamelCase_							=0
              with open(lowerCAmelCase,	'''w''',	encoding='''utf-8'''						) as writer:
                     writer.write('''#version: 0.2\n'''						)
                     for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),	key=lambda lowerCAmelCase						: kv[1]						):
                            if index != token_index:
                                   logger.warning(
                                       f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                       ''' Please check that the tokenizer is not corrupted!'''						)
                                   lowerCamelCase_							=token_index
                            writer.write(''' '''.join(lowerCAmelCase						) + '''\n'''						)
                            index += 1

              return vocab_file, merge_file






       def   lowercase__  (							self,	lowerCAmelCase,	lowerCAmelCase = None						):



              """simple docstring"""







              if token_ids_a is None:
                     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
              lowerCamelCase_							=[self.cls_token_id]
              lowerCamelCase_							=[self.sep_token_id]
              return cls + token_ids_a + sep + sep + token_ids_a + sep






       def   lowercase__  (							self,	lowerCAmelCase,	lowerCAmelCase = None,	lowerCAmelCase = False						):



              """simple docstring"""







              if already_has_special_tokens:
                     return super().get_special_tokens_mask(
                         token_ids_a=lowerCAmelCase,	token_ids_a=lowerCAmelCase,	already_has_special_tokens=lowerCAmelCase						)

              if token_ids_a is None:
                     return [1] + ([0] * len(lowerCAmelCase						)) + [1]
              return [1] + ([0] * len(lowerCAmelCase						)) + [1, 1] + ([0] * len(lowerCAmelCase						)) + [1]






       def   lowercase__  (							self,	lowerCAmelCase,	lowerCAmelCase = None						):



              """simple docstring"""







              lowerCamelCase_							=[self.sep_token_id]
              lowerCamelCase_							=[self.cls_token_id]

              if token_ids_a is None:
                     return len(cls + token_ids_a + sep						) * [0]
              return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]






       def   lowercase__  (							self,	lowerCAmelCase,	lowerCAmelCase=False,	**lowerCAmelCase						):



              """simple docstring"""







              lowerCamelCase_							=kwargs.pop('''add_prefix_space''',	self.add_prefix_space						)
              if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase						) > 0 and not text[0].isspace()):
                     lowerCamelCase_							=''' ''' + text
              return (text, kwargs)










'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a_   :	Union[str, Any]							   = {
    """configuration_openai""": ["""OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """OpenAIGPTConfig"""],
    """tokenization_openai""": ["""OpenAIGPTTokenizer"""],
}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     a_   :	Optional[int]							   = ["""OpenAIGPTTokenizerFast"""]

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     a_   :	int							   = [
         """OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """OpenAIGPTDoubleHeadsModel""",
         """OpenAIGPTForSequenceClassification""",
         """OpenAIGPTLMHeadModel""",
         """OpenAIGPTModel""",
         """OpenAIGPTPreTrainedModel""",
         """load_tf_weights_in_openai_gpt""",
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     a_   :	Dict							   = [
         """TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """TFOpenAIGPTDoubleHeadsModel""",
         """TFOpenAIGPTForSequenceClassification""",
         """TFOpenAIGPTLMHeadModel""",
         """TFOpenAIGPTMainLayer""",
         """TFOpenAIGPTModel""",
         """TFOpenAIGPTPreTrainedModel""",
     ]


if TYPE_CHECKING:
     from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig
     from .tokenization_openai import OpenAIGPTTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_openai_fast import OpenAIGPTTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_openai import (
              OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
              OpenAIGPTDoubleHeadsModel,
              OpenAIGPTForSequenceClassification,
              OpenAIGPTLMHeadModel,
              OpenAIGPTModel,
              OpenAIGPTPreTrainedModel,
              load_tf_weights_in_openai_gpt,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_openai import (
              TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFOpenAIGPTDoubleHeadsModel,
              TFOpenAIGPTForSequenceClassification,
              TFOpenAIGPTLMHeadModel,
              TFOpenAIGPTMainLayer,
              TFOpenAIGPTModel,
              TFOpenAIGPTPreTrainedModel,
          )

else:
     import sys

     a_   :	List[Any]							   = _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




