


'''simple docstring'''

import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
 if is_torch_available():
  import torch
 if is_tf_available():
  import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


lowerCamelCase							:     Union[str, Any]								=			logging.get_logger(__name__)

lowerCamelCase							:     Tuple								=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCamelCase							:     Tuple								=			{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

lowerCamelCase							:     Optional[int]								=			{
    'Salesforce/codegen-350M-mono': 2_048,
}

@lru_cache()
def _SCREAMING_SNAKE_CASE   () -> Optional[int]:






       """simple docstring"""







       lowercase__       =				(
           list(range(ord('''!''' )						,				ord('''~''' ) + 1 ) ) + list(range(ord('''¡''' )						,				ord('''¬''' ) + 1 ) ) + list(range(ord('''®''' )						,				ord('''ÿ''' ) + 1 ) )
       )
       lowercase__       =				bs[:]
       lowercase__       =				0
       for b in range(2**8 ):
              if b not in bs:
                     bs.append(A )
                     cs.append(2**8 + n )
                     n += 1
       lowercase__       =				[chr(A ) for n in cs]
       return dict(zip(A						,				A ) )

def _SCREAMING_SNAKE_CASE   (A ) -> Tuple:






       """simple docstring"""







       lowercase__       =				set()
       lowercase__       =				word[0]
       for char in word[1:]:
              pairs.add((prev_char, char) )
              lowercase__       =				char
       return pairs

class 						__lowerCAmelCase		(lowercase_  ):





      '''simple docstring'''






      lowerCAmelCase__      :  int						      =							VOCAB_FILES_NAMES
      lowerCAmelCase__      :  Union[str, Any]						      =							PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__      :  Optional[int]						      =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__      :  Tuple						      =							["""input_ids""", """attention_mask"""]



      def __init__(self					:					List[Any]     ,		UpperCamelCase					:					List[str]     ,		UpperCamelCase					:					List[Any]     ,		UpperCamelCase					:					Optional[int]="replace"     ,		UpperCamelCase					:					Optional[Any]="<|endoftext|>"     ,		UpperCamelCase					:					str="<|endoftext|>"     ,		UpperCamelCase					:					Optional[int]="<|endoftext|>"     ,		UpperCamelCase					:					Optional[int]=None     ,		UpperCamelCase					:					str=False     ,		UpperCamelCase					:					Any=False     ,		**UpperCamelCase					:					Dict     ,		):

             '''simple docstring'''







             lowercase__       =				AddedToken(UpperCamelCase     ,		lstrip=UpperCamelCase     ,		rstrip=UpperCamelCase				) if isinstance(UpperCamelCase     ,		UpperCamelCase				) else bos_token
             lowercase__       =				AddedToken(UpperCamelCase     ,		lstrip=UpperCamelCase     ,		rstrip=UpperCamelCase				) if isinstance(UpperCamelCase     ,		UpperCamelCase				) else eos_token
             lowercase__       =				AddedToken(UpperCamelCase     ,		lstrip=UpperCamelCase     ,		rstrip=UpperCamelCase				) if isinstance(UpperCamelCase     ,		UpperCamelCase				) else unk_token
             lowercase__       =				AddedToken(UpperCamelCase     ,		lstrip=UpperCamelCase     ,		rstrip=UpperCamelCase				) if isinstance(UpperCamelCase     ,		UpperCamelCase				) else pad_token
             super().__init__(
                 errors=UpperCamelCase     ,		unk_token=UpperCamelCase     ,		bos_token=UpperCamelCase     ,		eos_token=UpperCamelCase     ,		pad_token=UpperCamelCase     ,		add_prefix_space=UpperCamelCase     ,		add_bos_token=UpperCamelCase     ,		**UpperCamelCase     ,		)
             lowercase__       =				add_bos_token

             with open(UpperCamelCase     ,		encoding='''utf-8'''				) as vocab_handle:
                    lowercase__       =				json.load(UpperCamelCase				)
             lowercase__       =				{v: k for k, v in self.encoder.items()}
             lowercase__       =				errors  # how to handle errors in decoding
             lowercase__       =				bytes_to_unicode()
             lowercase__       =				{v: k for k, v in self.byte_encoder.items()}
             with open(UpperCamelCase     ,		encoding='''utf-8'''				) as merges_handle:
                    lowercase__       =				merges_handle.read().split('''\n'''				)[1:-1]
             lowercase__       =				[tuple(merge.split()				) for merge in bpe_merges]
             lowercase__       =				dict(zip(UpperCamelCase     ,		range(len(UpperCamelCase				)				)				)				)
             lowercase__       =				{}
             lowercase__       =				add_prefix_space

             # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
             lowercase__       =				re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''				)



      @property
      def    UpperCamelCase__	(self					:					Any				):

             '''simple docstring'''







             return len(self.encoder				)



      def    UpperCamelCase__	(self					:					Optional[Any]				):

             '''simple docstring'''







             return dict(self.encoder     ,		**self.added_tokens_encoder				)



      def    UpperCamelCase__	(self					:					Union[str, Any]     ,		UpperCamelCase					:					Optional[int]				):

             '''simple docstring'''







             if token in self.cache:
                    return self.cache[token]
             lowercase__       =				tuple(UpperCamelCase				)
             lowercase__       =				get_pairs(UpperCamelCase				)

             if not pairs:
                    return token

             while True:
                    lowercase__       =				min(UpperCamelCase     ,		key=lambda UpperCamelCase				: self.bpe_ranks.get(UpperCamelCase     ,		float('''inf'''				)				)				)
                    if bigram not in self.bpe_ranks:
                           break
                    lowercase__    ,lowercase__       =				bigram
                    lowercase__       =				[]
                    lowercase__       =				0
                    while i < len(UpperCamelCase				):
                           try:
                                  lowercase__       =				word.index(UpperCamelCase     ,		UpperCamelCase				)
                           except ValueError:
                                  new_word.extend(word[i:]				)
                                  break
                           else:
                                  new_word.extend(word[i:j]				)
                                  lowercase__       =				j

                           if word[i] == first and i < len(UpperCamelCase				) - 1 and word[i + 1] == second:
                                  new_word.append(first + second				)
                                  i += 2
                           else:
                                  new_word.append(word[i]				)
                                  i += 1
                    lowercase__       =				tuple(UpperCamelCase				)
                    lowercase__       =				new_word
                    if len(UpperCamelCase				) == 1:
                           break
                    else:
                           lowercase__       =				get_pairs(UpperCamelCase				)
             lowercase__       =				''' '''.join(UpperCamelCase				)
             lowercase__       =				word
             return word



      def    UpperCamelCase__	(self					:					Any     ,		UpperCamelCase					:					str     ,		UpperCamelCase					:					str=None				):

             '''simple docstring'''







             if self.add_bos_token:
                    lowercase__       =				[self.bos_token_id]
             else:
                    lowercase__       =				[]

             lowercase__       =				bos_token_ids + token_ids_a

             if token_ids_a is None:
                    return output

             return output + bos_token_ids + token_ids_a



      def    UpperCamelCase__	(self					:					Optional[Any]     ,		UpperCamelCase					:					str				):

             '''simple docstring'''







             lowercase__       =				[]
             for token in re.findall(self.pat     ,		UpperCamelCase				):
                    lowercase__       =				''''''.join(
                        self.byte_encoder[b] for b in token.encode('''utf-8'''				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                    bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase				).split(''' '''				)				)
             return bpe_tokens



      def    UpperCamelCase__	(self					:					str     ,		UpperCamelCase					:					str				):

             '''simple docstring'''







             return self.encoder.get(UpperCamelCase     ,		self.encoder.get(self.unk_token				)				)



      def    UpperCamelCase__	(self					:					Any     ,		UpperCamelCase					:					Optional[int]				):

             '''simple docstring'''







             return self.decoder.get(UpperCamelCase				)



      def    UpperCamelCase__	(self					:					List[Any]     ,		UpperCamelCase					:					Optional[int]				):

             '''simple docstring'''







             lowercase__       =				''''''.join(UpperCamelCase				)
             lowercase__       =				bytearray([self.byte_decoder[c] for c in text]				).decode('''utf-8'''     ,		errors=self.errors				)
             return text



      def    UpperCamelCase__	(self					:					int     ,		UpperCamelCase					:					str     ,		UpperCamelCase					:					Optional[str] = None				):

             '''simple docstring'''







             if not os.path.isdir(UpperCamelCase				):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory"				)
                    return
             lowercase__       =				os.path.join(
                 UpperCamelCase     ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
             lowercase__       =				os.path.join(
                 UpperCamelCase     ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']				)

             with open(UpperCamelCase     ,		'''w'''     ,		encoding='''utf-8'''				) as f:
                    f.write(json.dumps(self.encoder     ,		indent=2     ,		sort_keys=UpperCamelCase     ,		ensure_ascii=UpperCamelCase				) + '''\n'''				)

             lowercase__       =				0
             with open(UpperCamelCase     ,		'''w'''     ,		encoding='''utf-8'''				) as writer:
                    writer.write('''#version: 0.2\n'''				)
                    for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,		key=lambda UpperCamelCase				: kv[1]				):
                           if index != token_index:
                                  logger.warning(
                                      f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                                      ''' Please check that the tokenizer is not corrupted!'''				)
                                  lowercase__       =				token_index
                           writer.write(''' '''.join(UpperCamelCase				) + '''\n'''				)
                           index += 1

             return vocab_file, merge_file



      def    UpperCamelCase__	(self					:					str     ,		UpperCamelCase					:					List[Any]     ,		UpperCamelCase					:					int=False     ,		**UpperCamelCase					:					Optional[int]				):

             '''simple docstring'''







             lowercase__       =				kwargs.pop('''add_prefix_space'''     ,		self.add_prefix_space				)
             if is_split_into_words or add_prefix_space:
                    lowercase__       =				''' ''' + text
             return (text, kwargs)



      def    UpperCamelCase__	(self					:					Tuple     ,		UpperCamelCase					:					Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]     ,		UpperCamelCase					:					bool = False     ,		UpperCamelCase					:					bool = None     ,		UpperCamelCase					:					Optional[List[str]] = None     ,		**UpperCamelCase					:					Optional[Any]     ,		):

             '''simple docstring'''







             lowercase__       =				super()._decode(
                 token_ids=UpperCamelCase     ,		skip_special_tokens=UpperCamelCase     ,		clean_up_tokenization_spaces=UpperCamelCase     ,		**UpperCamelCase     ,		)

             if truncate_before_pattern is not None and len(UpperCamelCase				) > 0:
                    lowercase__       =				self.truncate(UpperCamelCase     ,		UpperCamelCase				)

             return decoded_text






      def    UpperCamelCase__	(self					:					str     ,		UpperCamelCase					:					List[str]     ,		UpperCamelCase					:					Union[str, Any]				):

             '''simple docstring'''







             def find_re(UpperCamelCase					:					Dict     ,		UpperCamelCase					:					Dict     ,		UpperCamelCase					:					Optional[Any]				):
                    lowercase__       =				pattern.search(UpperCamelCase     ,		UpperCamelCase				)
                    return m.start() if m else -1

             lowercase__       =				[re.compile(UpperCamelCase     ,		re.MULTILINE				) for pattern in truncate_before_pattern]

             lowercase__       =				list(re.finditer('''^print'''     ,		UpperCamelCase     ,		re.MULTILINE				)				)

             if len(UpperCamelCase				) > 1:
                    lowercase__       =				completion[: prints[1].start()]

             lowercase__       =				list(re.finditer('''^def'''     ,		UpperCamelCase     ,		re.MULTILINE				)				)

             if len(UpperCamelCase				) > 1:
                    lowercase__       =				completion[: defs[1].start()]

             lowercase__       =				0

             lowercase__       =				[
                 pos for pos in [find_re(UpperCamelCase     ,		UpperCamelCase     ,		UpperCamelCase				) for terminal in terminals] if pos != -1
             ]

             if len(UpperCamelCase				) > 0:
                    return completion[: min(UpperCamelCase				)]
             else:
                    return completion








'''simple docstring'''

import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation

require_version('tokenizers>=0.13.3')

if is_sentencepiece_available():
 from .tokenization_llama import LlamaTokenizer
else:
 lowerCamelCase							:     Optional[int]								=			None

lowerCamelCase							:     List[Any]								=			logging.get_logger(__name__)
lowerCamelCase							:     Dict								=			{'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase		,						lowerCamelCase							:     str								=			'[INST]', '[/INST]'
lowerCamelCase		,						lowerCamelCase							:     Optional[Any]								=			'<<SYS>>\n', '\n<</SYS>>\n\n'

# fmt: off
lowerCamelCase							:     Optional[int]								=			'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'
class 						__lowerCAmelCase		(lowercase_  ):





      '''simple docstring'''






      lowerCAmelCase__      :  int						      =							VOCAB_FILES_NAMES
      lowerCAmelCase__      :  List[str]						      =							LlamaTokenizer
      lowerCAmelCase__      :  Tuple						      =							"""left"""
      lowerCAmelCase__      :  int						      =							["""input_ids""", """attention_mask"""]



      def __init__(self					:					Optional[Any]     ,		UpperCamelCase					:					Tuple=None     ,		UpperCamelCase					:					Optional[int]=None     ,		UpperCamelCase					:					List[str]=False     ,		UpperCamelCase					:					Dict="<unk>"     ,		UpperCamelCase					:					int="<s>"     ,		UpperCamelCase					:					Dict="</s>"     ,		UpperCamelCase					:					Any=True     ,		UpperCamelCase					:					Tuple=False     ,		**UpperCamelCase					:					List[str]     ,		):

             '''simple docstring'''







             super().__init__(
                 vocab_file=UpperCamelCase     ,		tokenizer_file=UpperCamelCase     ,		clean_up_tokenization_spaces=UpperCamelCase     ,		unk_token=UpperCamelCase     ,		bos_token=UpperCamelCase     ,		eos_token=UpperCamelCase     ,		**UpperCamelCase     ,		)
             lowercase__       =				add_bos_token
             lowercase__       =				add_eos_token
             self.update_post_processor()

             lowercase__       =				vocab_file
             lowercase__       =				False if not self.vocab_file else True



      def    UpperCamelCase__	(self					:					Tuple				):

             '''simple docstring'''







             lowercase__       =				self.bos_token
             lowercase__       =				self.bos_token_id

             lowercase__       =				self.eos_token
             lowercase__       =				self.eos_token_id

             lowercase__       =				f"{(bos+':0 ') * self.add_bos_token}$A:0{(' '+eos+':0') * self.add_eos_token}"
             lowercase__       =				f"{single}{(' '+bos+':1') * self.add_bos_token} $B:1{(' '+eos+':1') * self.add_eos_token}"

             lowercase__       =				[]
             if self.add_bos_token:
                    special_tokens.append((bos, bos_token_id)				)
             if self.add_eos_token:
                    special_tokens.append((eos, eos_token_id)				)
             lowercase__       =				processors.TemplateProcessing(
                 single=UpperCamelCase     ,		pair=UpperCamelCase     ,		special_tokens=UpperCamelCase				)



      @property
      def    UpperCamelCase__	(self					:					Union[str, Any]				):

             '''simple docstring'''







             return self._add_eos_token



      @property
      def    UpperCamelCase__	(self					:					Tuple				):

             '''simple docstring'''







             return self._add_bos_token



      @add_eos_token.setter
      def    UpperCamelCase__	(self					:					Union[str, Any]     ,		UpperCamelCase					:					int				):

             '''simple docstring'''







             lowercase__       =				value
             self.update_post_processor()



      @add_bos_token.setter
      def    UpperCamelCase__	(self					:					Optional[Any]     ,		UpperCamelCase					:					Optional[int]				):

             '''simple docstring'''







             lowercase__       =				value
             self.update_post_processor()



      def    UpperCamelCase__	(self					:					Optional[Any]     ,		UpperCamelCase					:					str     ,		UpperCamelCase					:					Optional[str] = None				):

             '''simple docstring'''







             if not self.can_save_slow_tokenizer:
                    raise ValueError(
                        '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                        '''tokenizer.'''				)

             if not os.path.isdir(UpperCamelCase				):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory"				)
                    return
             lowercase__       =				os.path.join(
                 UpperCamelCase     ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

             if os.path.abspath(self.vocab_file				) != os.path.abspath(UpperCamelCase				):
                    copyfile(self.vocab_file     ,		UpperCamelCase				)

             return (out_vocab_file,)






      def    UpperCamelCase__	(self					:					str     ,		UpperCamelCase					:					"Conversation"				):

             '''simple docstring'''







             lowercase__       =				list(conversation.iter_texts()				)
             if not all([is_user for is_user, msg in dialogue[::2]]				) or not all(
                 [not is_user for is_user, msg in dialogue[1::2]]				):
                    raise ValueError(
                        '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)'''				)

             lowercase__       =				[]
             if len(conversation.past_user_inputs				) > 0:
                    if not conversation.past_user_inputs[0].startswith(UpperCamelCase				) or E_SYS not in conversation.past_user_inputs[0]:
                           lowercase__       =				(
                               B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
                           )
             elif not dialogue[0][1].startswith(UpperCamelCase				) or E_SYS not in dialogue[0][1]:
                    lowercase__       =				(dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

             dialog_tokens += sum(
                 [
                     [self.bos_token_id]
                     + self.encode(
                         f"{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} "     ,		add_special_tokens=UpperCamelCase				)
                     + [self.eos_token_id]
                     for prompt, answer in zip(dialogue[::2]     ,		dialogue[1::2]				)
                 ]     ,		[]     ,		)
             if not (dialogue[-1][0]):
                    raise ValueError(f"Last message must be from user, got {dialogue[-1]['role']}"				)
             dialog_tokens += [self.bos_token_id] + self.encode(
                 f"{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"     ,		add_special_tokens=UpperCamelCase				)
             return dialog_tokens




