






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_UpperCAmelCase					:						Any         =     logging.get_logger(__name__)

_UpperCAmelCase					:						Optional[Any]         =     {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_UpperCAmelCase					:						int         =     {
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

_UpperCAmelCase					:						Optional[int]         =     {
    '''openai-gpt''': 512,
}






def  UpperCAmelCase__						(    lowerCamelCase	:			List[str]    ):
   lowerCamelCase									=text.strip()
   if not text:
      return []
   lowerCamelCase									=text.split()
   return tokens


class 		lowerCAmelCase_				(  UpperCamelCase ):





     def __init__(	self							, _UpperCAmelCase=True							, _UpperCAmelCase=None							, _UpperCAmelCase=True							, _UpperCAmelCase=None							, _UpperCAmelCase=True							, )		->				Union[str, Any]:






        if never_split is None:
           lowerCamelCase									=[]
        lowerCamelCase									=do_lower_case
        lowerCamelCase									=set(_UpperCAmelCase  )
        lowerCamelCase									=tokenize_chinese_chars
        lowerCamelCase									=strip_accents
        lowerCamelCase									=do_split_on_punc





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase=None  )		->				Any:
        lowerCamelCase									=self.never_split.union(set(_UpperCAmelCase  )  ) if never_split else self.never_split
        lowerCamelCase									=self._clean_text(_UpperCAmelCase  )

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        if self.tokenize_chinese_chars:
           lowerCamelCase									=self._tokenize_chinese_chars(_UpperCAmelCase  )
        # prevents treating the same character with different unicode codepoints as different characters
        lowerCamelCase									=unicodedata.normalize("""NFC"""							, _UpperCAmelCase  )
        lowerCamelCase									=whitespace_tokenize(_UpperCAmelCase  )
        lowerCamelCase									=[]
        for token in orig_tokens:
           if token not in never_split:
              if self.do_lower_case:
                 lowerCamelCase									=token.lower()
                 if self.strip_accents is not False:
                    lowerCamelCase									=self._run_strip_accents(_UpperCAmelCase  )
              elif self.strip_accents:
                 lowerCamelCase									=self._run_strip_accents(_UpperCAmelCase  )
           split_tokens.extend(self._run_split_on_punc(_UpperCAmelCase							, _UpperCAmelCase  )  )

        lowerCamelCase									=whitespace_tokenize(""" """.join(_UpperCAmelCase  )  )
        return output_tokens





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Tuple:
        lowerCamelCase									=unicodedata.normalize("""NFD"""							, _UpperCAmelCase  )
        lowerCamelCase									=[]
        for char in text:
           lowerCamelCase									=unicodedata.category(_UpperCAmelCase  )
           if cat == "Mn":
              continue
           output.append(_UpperCAmelCase  )
        return "".join(_UpperCAmelCase  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase=None  )		->				Optional[int]:






        if not self.do_split_on_punc or (never_split is not None and text in never_split):
           return [text]
        lowerCamelCase									=list(_UpperCAmelCase  )
        lowerCamelCase									=0
        lowerCamelCase									=True
        lowerCamelCase									=[]
        while i < len(_UpperCAmelCase  ):
           lowerCamelCase									=chars[i]
           if _is_punctuation(_UpperCAmelCase  ):
              output.append([char]  )
              lowerCamelCase									=True
           else:
              if start_new_word:
                 output.append([]  )
              lowerCamelCase									=False
              output[-1].append(_UpperCAmelCase  )
           i += 1

        return ["".join(_UpperCAmelCase  ) for x in output]





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Optional[Any]:
        lowerCamelCase									=[]
        for char in text:
           lowerCamelCase									=ord(_UpperCAmelCase  )
           if self._is_chinese_char(_UpperCAmelCase  ):
              output.append(""" """  )
              output.append(_UpperCAmelCase  )
              output.append(""" """  )
           else:
              output.append(_UpperCAmelCase  )
        return "".join(_UpperCAmelCase  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Optional[Any]:






        if (
            (cp >= 0x4E00 and cp <= 0x9FFF)
            or (cp >= 0x3400 and cp <= 0x4DBF)  #
            or (cp >= 0x20000 and cp <= 0x2A6DF)  #
            or (cp >= 0x2A700 and cp <= 0x2B73F)  #
            or (cp >= 0x2B740 and cp <= 0x2B81F)  #
            or (cp >= 0x2B820 and cp <= 0x2CEAF)  #
            or (cp >= 0xF900 and cp <= 0xFAFF)
            or (cp >= 0x2F800 and cp <= 0x2FA1F)  #
        ):  #
           return True

        return False





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Dict:
        lowerCamelCase									=[]
        for char in text:
           lowerCamelCase									=ord(_UpperCAmelCase  )
           if cp == 0 or cp == 0xFFFD or _is_control(_UpperCAmelCase  ):
              continue
           if _is_whitespace(_UpperCAmelCase  ):
              output.append(""" """  )
           else:
              output.append(_UpperCAmelCase  )
        return "".join(_UpperCAmelCase  )






def  UpperCAmelCase__						(    lowerCamelCase	:			Optional[int]    ):
   lowerCamelCase									=set()
   lowerCamelCase									=word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)    )
      lowerCamelCase									=char
   return pairs






def  UpperCAmelCase__						(    lowerCamelCase	:			int    ):
   lowerCamelCase									=text.replace("""—"""			,"""-"""    )
   lowerCamelCase									=text.replace("""–"""			,"""-"""    )
   lowerCamelCase									=text.replace("""―"""			,"""-"""    )
   lowerCamelCase									=text.replace("""…"""			,"""..."""    )
   lowerCamelCase									=text.replace("""´"""			,"""\'"""    )
   lowerCamelCase									=re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""			,R""" \1 """			,lowerCamelCase    )
   lowerCamelCase									=re.sub(R"""\s*\n\s*"""			,""" \n """			,lowerCamelCase    )
   lowerCamelCase									=re.sub(R"""[^\S\n]+"""			,""" """			,lowerCamelCase    )
   return text.strip()







class 		lowerCAmelCase_				(  UpperCamelCase ):
     __lowerCamelCase       :    Union[str, Any] 	=     VOCAB_FILES_NAMES
     __lowerCamelCase       :    List[Any] 	=     PRETRAINED_VOCAB_FILES_MAP
     __lowerCamelCase       :    Optional[Any] 	=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __lowerCamelCase       :    str 	=     ['input_ids', 'attention_mask']





     def __init__(	self							, _UpperCAmelCase							, _UpperCAmelCase							, _UpperCAmelCase="<unk>"							, **_UpperCAmelCase  )		->				Union[str, Any]:
        super().__init__(unk_token=_UpperCAmelCase							, **_UpperCAmelCase  )

        try:
           import ftfy
           from spacy.lang.en import English

           lowerCamelCase									=English()
           lowerCamelCase									=_nlp.tokenizer
           lowerCamelCase									=ftfy.fix_text
        except ImportError:
           logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""  )
           lowerCamelCase									=BasicTokenizer(do_lower_case=_UpperCAmelCase  )
           lowerCamelCase									=None

        with open(_UpperCAmelCase							, encoding="""utf-8"""  ) as vocab_handle:
           lowerCamelCase									=json.load(_UpperCAmelCase  )
        lowerCamelCase									={v: k for k, v in self.encoder.items()}
        with open(_UpperCAmelCase							, encoding="""utf-8"""  ) as merges_handle:
           lowerCamelCase									=merges_handle.read().split("""\n"""  )[1:-1]
        lowerCamelCase									=[tuple(merge.split()  ) for merge in merges]
        lowerCamelCase									=dict(zip(_UpperCAmelCase							, range(len(_UpperCAmelCase  )  )  )  )
        lowerCamelCase									={}





     @property
     def 						lowerCamelCase							(	self  )		->				Union[str, Any]:
        return True





     @property
     def 						lowerCamelCase							(	self  )		->				Optional[int]:
        return len(self.encoder  )





     def 						lowerCamelCase							(	self  )		->				Union[str, Any]:
        return dict(self.encoder							, **self.added_tokens_encoder  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				str:
        lowerCamelCase									=tuple(token[:-1]  ) + (token[-1] + """</w>""",)
        if token in self.cache:
           return self.cache[token]
        lowerCamelCase									=get_pairs(_UpperCAmelCase  )

        if not pairs:
           return token + "</w>"

        while True:
           lowerCamelCase									=min(_UpperCAmelCase							, key=lambda _UpperCAmelCase  : self.bpe_ranks.get(_UpperCAmelCase							, float("""inf"""  )  )  )
           if bigram not in self.bpe_ranks:
              break
           lowerCamelCase				,       lowerCamelCase									=bigram
           lowerCamelCase									=[]
           lowerCamelCase									=0
           while i < len(_UpperCAmelCase  ):
              try:
                 lowerCamelCase									=word.index(_UpperCAmelCase							, _UpperCAmelCase  )
              except ValueError:
                 new_word.extend(word[i:]  )
                 break
              else:
                 new_word.extend(word[i:j]  )
                 lowerCamelCase									=j

              if word[i] == first and i < len(_UpperCAmelCase  ) - 1 and word[i + 1] == second:
                 new_word.append(first + second  )
                 i += 2
              else:
                 new_word.append(word[i]  )
                 i += 1
           lowerCamelCase									=tuple(_UpperCAmelCase  )
           lowerCamelCase									=new_word
           if len(_UpperCAmelCase  ) == 1:
              break
           else:
              lowerCamelCase									=get_pairs(_UpperCAmelCase  )
        lowerCamelCase									=""" """.join(_UpperCAmelCase  )
        if word == "\n  </w>":
           lowerCamelCase									="""\n</w>"""
        lowerCamelCase									=word
        return word





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Optional[Any]:
        lowerCamelCase									=[]
        if self.fix_text is None:
           # Using BERT's BasicTokenizer
           lowerCamelCase									=self.nlp.tokenize(_UpperCAmelCase  )
           for token in text:
              split_tokens.extend(list(self.bpe(_UpperCAmelCase  ).split(""" """  )  )  )
        else:
           # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
           lowerCamelCase									=self.nlp(text_standardize(self.fix_text(_UpperCAmelCase  )  )  )
           for token in text:
              split_tokens.extend(list(self.bpe(token.text.lower()  ).split(""" """  )  )  )
        return split_tokens





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Union[str, Any]:
        return self.encoder.get(_UpperCAmelCase							, self.encoder.get(self.unk_token  )  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Optional[Any]:
        return self.decoder.get(_UpperCAmelCase							, self.unk_token  )





     def 						lowerCamelCase							(	self							, _UpperCAmelCase  )		->				Optional[Any]:
        lowerCamelCase									="""""".join(_UpperCAmelCase  ).replace("""</w>"""							, """ """  ).strip()
        return out_string





     def 						lowerCamelCase							(	self							, _UpperCAmelCase							, _UpperCAmelCase = None  )		->				Tuple[str]:






        if not os.path.isdir(_UpperCAmelCase  ):
           logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
           return
        lowerCamelCase									=os.path.join(
            _UpperCAmelCase							, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
        lowerCamelCase									=os.path.join(
            _UpperCAmelCase							, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

        with open(_UpperCAmelCase							, """w"""							, encoding="""utf-8"""  ) as f:
           f.write(json.dumps(self.encoder							, indent=2							, sort_keys=_UpperCAmelCase							, ensure_ascii=_UpperCAmelCase  ) + """\n"""  )

        lowerCamelCase									=0
        with open(_UpperCAmelCase							, """w"""							, encoding="""utf-8"""  ) as writer:
           writer.write("""#version: 0.2\n"""  )
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							, key=lambda _UpperCAmelCase  : kv[1]  ):
              if index != token_index:
                 logger.warning(
                     F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                     """ Please check that the tokenizer is not corrupted!"""  )
                 lowerCamelCase									=token_index
              writer.write(""" """.join(_UpperCAmelCase  ) + """\n"""  )
              index += 1

        return vocab_file, merge_file

"""simple docstring"""




import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation

require_version('''tokenizers>=0.13.3''')

if is_sentencepiece_available():
					from .tokenization_llama import LlamaTokenizer
else:
					__lowerCAmelCase							:  List[str] 				=		None

__lowerCAmelCase							:  Optional[int] 				=		logging.get_logger(__name__)
__lowerCAmelCase							:  int 				=		{'''vocab_file''': '''tokenizer.model''', '''tokenizer_file''': '''tokenizer.json'''}

__lowerCAmelCase				,__lowerCAmelCase							:  Any 				=		'''[INST]''', '''[/INST]'''
__lowerCAmelCase				,__lowerCAmelCase							:  Tuple 				=		'''<<SYS>>\n''', '''\n<</SYS>>\n\n'''

# fmt: off
__lowerCAmelCase							:  List[str] 				=		'''You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
correct. If you don\'t know the answer to a question, please don\'t share false information.'''
class SCREAMING_SNAKE_CASE       (      lowerCAmelCase					):






	'''simple docstring'''







	snake_case__							:		List[str]         =		VOCAB_FILES_NAMES
	snake_case__							:		Optional[Any]         =		LlamaTokenizer
	snake_case__							:		Union[str, Any]         =		'left'
	snake_case__							:		Any         =		['input_ids', 'attention_mask']


	def __init__(  self :Optional[Any]       ,    __magic_name__ :Optional[Any]=None       ,    __magic_name__ :Tuple=None       ,    __magic_name__ :List[Any]=False       ,    __magic_name__ :List[str]="<unk>"       ,    __magic_name__ :List[str]="<s>"       ,    __magic_name__ :List[str]="</s>"       ,    __magic_name__ :Any=True       ,    __magic_name__ :Union[str, Any]=False       ,    **__magic_name__ :Optional[int]       ,    )			->   List[Any]:

			'''simple docstring'''






			super().__init__(
			    vocab_file=__magic_name__       ,    tokenizer_file=__magic_name__       ,    clean_up_tokenization_spaces=__magic_name__       ,    unk_token=__magic_name__       ,    bos_token=__magic_name__       ,    eos_token=__magic_name__       ,    **__magic_name__       ,    )
			a__   		=     add_bos_token
			a__   		=     add_eos_token
			self.update_post_processor()

			a__   		=     vocab_file
			a__   		=     False if not self.vocab_file else True


	def    _UpperCamelCase							(  self :List[str]			)			->   Optional[int]:

			'''simple docstring'''






			a__   		=     self.bos_token
			a__   		=     self.bos_token_id

			a__   		=     self.eos_token
			a__   		=     self.eos_token_id

			a__   		=     F"{(bos+':0 ') * self.add_bos_token}$A:0{(' '+eos+':0') * self.add_eos_token}"
			a__   		=     F"{single}{(' '+bos+':1') * self.add_bos_token} $B:1{(' '+eos+':1') * self.add_eos_token}"

			a__   		=     []
			if self.add_bos_token:
					special_tokens.append((bos, bos_token_id)			)
			if self.add_eos_token:
					special_tokens.append((eos, eos_token_id)			)
			a__   		=     processors.TemplateProcessing(
			    single=__magic_name__       ,    pair=__magic_name__       ,    special_tokens=__magic_name__			)


	@property
	def    _UpperCamelCase							(  self :Optional[int]			)			->   Any:

			'''simple docstring'''






			return self._add_eos_token


	@property
	def    _UpperCamelCase							(  self :Optional[int]			)			->   Optional[Any]:

			'''simple docstring'''






			return self._add_bos_token


	@add_eos_token.setter
	def    _UpperCamelCase							(  self :int       ,    __magic_name__ :int			)			->   int:

			'''simple docstring'''






			a__   		=     value
			self.update_post_processor()


	@add_bos_token.setter
	def    _UpperCamelCase							(  self :Any       ,    __magic_name__ :str			)			->   Optional[int]:

			'''simple docstring'''






			a__   		=     value
			self.update_post_processor()


	def    _UpperCamelCase							(  self :Dict       ,    __magic_name__ :str       ,    __magic_name__ :Optional[str] = None			)			->   Tuple[str]:

			'''simple docstring'''






			if not self.can_save_slow_tokenizer:
					raise ValueError(
					    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
					    '''tokenizer.'''			)

			if not os.path.isdir(__magic_name__			):
					logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
					return
			a__   		=     os.path.join(
			    __magic_name__       ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)

			if os.path.abspath(self.vocab_file			) != os.path.abspath(__magic_name__			):
					copyfile(self.vocab_file       ,    __magic_name__			)

			return (out_vocab_file,)






	def    _UpperCamelCase							(  self :int       ,    __magic_name__ :"Conversation"			)			->   List[Any]:

			'''simple docstring'''






			a__   		=     list(conversation.iter_texts()			)
			if not all([is_user for is_user, msg in dialogue[::2]]			) or not all(
			    [not is_user for is_user, msg in dialogue[1::2]]			):
					raise ValueError(
					    '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''			)

			a__   		=     []
			if len(conversation.past_user_inputs			) > 0:
					if not conversation.past_user_inputs[0].startswith(__magic_name__			) or E_SYS not in conversation.past_user_inputs[0]:
							a__   		=     (
							    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
							)
			elif not dialogue[0][1].startswith(__magic_name__			) or E_SYS not in dialogue[0][1]:
					a__   		=     (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

			dialog_tokens += sum(
			    [
			        [self.bos_token_id]
			        + self.encode(
			            F"{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} "       ,    add_special_tokens=__magic_name__			)
			        + [self.eos_token_id]
			        for prompt, answer in zip(dialogue[::2]       ,    dialogue[1::2]			)
			    ]       ,    []       ,    )
			if not (dialogue[-1][0]):
					raise ValueError(F"Last message must be from user, got {dialogue[-1]['role']}"			)
			dialog_tokens += [self.bos_token_id] + self.encode(
			    F"{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"       ,    add_special_tokens=__magic_name__			)
			return dialog_tokens




