
"""simple docstring"""






import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation

require_version('tokenizers>=0.13.3')

if is_sentencepiece_available():
      from .tokenization_llama import LlamaTokenizer
else:
      UpperCamelCase_						   =   None

UpperCamelCase_						   =   logging.get_logger(__name__)
UpperCamelCase_						   =   {"""vocab_file""": """tokenizer.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase_						   =   """[INST]""", """[/INST]"""
UpperCamelCase_						   =   """<<SYS>>\n""", """\n<</SYS>>\n\n"""

# fmt: off
UpperCamelCase_						   =   """You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
correct. If you don't know the answer to a question, please don't share false information."""





class 							snake_case       (     __UpperCAmelCase		):
   a_					:		Tuple     					=     VOCAB_FILES_NAMES
   a_					:		Dict     					=     LlamaTokenizer
   a_					:		Optional[int]     					=     """left"""
   a_					:		Union[str, Any]     					=     ["""input_ids""", """attention_mask"""]







   def __init__( self       ,							__UpperCAmelCase=None       ,							__UpperCAmelCase=None       ,							__UpperCAmelCase=False       ,							__UpperCAmelCase="<unk>"       ,							__UpperCAmelCase="<s>"       ,							__UpperCAmelCase="</s>"       ,							__UpperCAmelCase=True       ,							__UpperCAmelCase=False       ,							**__UpperCAmelCase       ,							)   ->str:
      super().__init__(
          vocab_file=__UpperCAmelCase       ,							tokenizer_file=__UpperCAmelCase       ,							clean_up_tokenization_spaces=__UpperCAmelCase       ,							unk_token=__UpperCAmelCase       ,							bos_token=__UpperCAmelCase       ,							eos_token=__UpperCAmelCase       ,							**__UpperCAmelCase       ,							)
      a_						   =     add_bos_token
      a_						   =     add_eos_token
      self.update_post_processor()

      a_						   =     vocab_file
      a_						   =     False if not self.vocab_file else True







   def    UpperCAmelCase__	( self)   ->List[Any]:
      a_						   =     self.bos_token
      a_						   =     self.bos_token_id

      a_						   =     self.eos_token
      a_						   =     self.eos_token_id

      a_						   =     F'''{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}'''
      a_						   =     F'''{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}'''

      a_						   =     []
      if self.add_bos_token:
         special_tokens.append((bos, bos_token_id))
      if self.add_eos_token:
         special_tokens.append((eos, eos_token_id))
      a_						   =     processors.TemplateProcessing(
          single=__UpperCAmelCase       ,							pair=__UpperCAmelCase       ,							special_tokens=__UpperCAmelCase)







   @property
   def    UpperCAmelCase__	( self)   ->Any:
      return self._add_eos_token







   @property
   def    UpperCAmelCase__	( self)   ->Any:
      return self._add_bos_token







   @add_eos_token.setter
   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->str:
      a_						   =     value
      self.update_post_processor()







   @add_bos_token.setter
   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->int:
      a_						   =     value
      self.update_post_processor()







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase       ,							__UpperCAmelCase = None)   ->Optional[int]:

      if not self.can_save_slow_tokenizer:
         raise ValueError(
             "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
             "tokenizer.")

      if not os.path.isdir(__UpperCAmelCase):
         logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
         return
      a_						   =     os.path.join(
          __UpperCAmelCase       ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

      if os.path.abspath(self.vocab_file) != os.path.abspath(__UpperCAmelCase):
         copyfile(self.vocab_file       ,							__UpperCAmelCase)

      return (out_vocab_file,)







   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->int:
      a_						   =     list(conversation.iter_texts())
      if not all([is_user for is_user, msg in dialogue[::2]]) or not all(
          [not is_user for is_user, msg in dialogue[1::2]]):
         raise ValueError(
             "The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)")

      a_						   =     []
      if len(conversation.past_user_inputs) > 0:
         if not conversation.past_user_inputs[0].startswith(__UpperCAmelCase) or E_SYS not in conversation.past_user_inputs[0]:
            a_						   =     (
                B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
            )
      elif not dialogue[0][1].startswith(__UpperCAmelCase) or E_SYS not in dialogue[0][1]:
         a_						   =     (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

      dialog_tokens += sum(
          [
              [self.bos_token_id]
              + self.encode(
                  F'''{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} '''       ,							add_special_tokens=__UpperCAmelCase)
              + [self.eos_token_id]
              for prompt, answer in zip(dialogue[::2]       ,							dialogue[1::2])
          ]       ,							[]       ,							)
      if not (dialogue[-1][0]):
         raise ValueError(F'''Last message must be from user, got {dialogue[-1]["role"]}''')
      dialog_tokens += [self.bos_token_id] + self.encode(
          F'''{B_INST} {(dialogue[-1][1]).strip()} {E_INST}'''       ,							add_special_tokens=__UpperCAmelCase)
      return dialog_tokens






import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available


def      SCREAMING_SNAKE_CASE      (   lowerCAmelCase     ,							lowerCAmelCase     ,							lowerCAmelCase     ,							lowerCAmelCase=None       ):
       _UpperCamelCase														=    None
       # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
       if str(weight.dtype       ) == "torch.bfloat16":
              # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
              _UpperCamelCase														=    weight.view(torch.intaa       )
              _UpperCamelCase														=    '''bfloat16'''
       _UpperCamelCase														=    weight.cpu().numpy()
       _UpperCamelCase														=    os.path.join(lowerCAmelCase     ,							f'''{weight_name}.dat'''       )
       if index is not None:
              if dtype is None:
                     _UpperCamelCase														=    str(array.dtype       )
              _UpperCamelCase														=    {'''dtype''': dtype, '''shape''': list(array.shape       )}
       if array.ndim == 0:
              _UpperCamelCase														=    array[None]
       _UpperCamelCase														=    np.memmap(lowerCAmelCase     ,							dtype=array.dtype     ,							mode='''w+'''     ,							shape=array.shape       )
       _UpperCamelCase														=    array[:]
       file_array.flush()
       return index


def      SCREAMING_SNAKE_CASE      (   lowerCAmelCase     ,							lowerCAmelCase       ):
       _UpperCamelCase														=    tuple(weight_info['''shape''']       )
       if shape == ():
              # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
              _UpperCamelCase														=    (1,)

       _UpperCamelCase														=    weight_info['''dtype''']
       if dtype == "bfloat16":
              # NumPy does not support bfloat16 so this was saved as a int16
              _UpperCamelCase														=    '''int16'''

       _UpperCamelCase														=    np.memmap(lowerCAmelCase     ,							dtype=lowerCAmelCase     ,							shape=lowerCAmelCase     ,							mode='''r'''       )

       if len(weight_info['''shape''']       ) == 0:
              _UpperCamelCase														=    weight[0]
       _UpperCamelCase														=    torch.tensor(lowerCAmelCase       )
       if weight_info["dtype"] == "bfloat16":
              _UpperCamelCase														=    weight.view(torch.bfloataa       )

       return weight


def      SCREAMING_SNAKE_CASE      (   lowerCAmelCase     ,							lowerCAmelCase       ):
       if index is None or len(lowerCAmelCase       ) == 0:
              # Nothing to save
              return

       _UpperCamelCase														=    os.path.join(lowerCAmelCase     ,							'''index.json'''       )
       if os.path.isfile(lowerCAmelCase       ):
              with open(lowerCAmelCase     ,							'''r'''     ,							encoding='''utf-8'''       ) as f:
                     _UpperCamelCase														=    json.load(lowerCAmelCase       )
       else:
              _UpperCamelCase														=    {}
       current_index.update(lowerCAmelCase       )

       with open(lowerCAmelCase     ,							'''w'''     ,							encoding='''utf-8'''       ) as f:
              json.dump(lowerCAmelCase     ,							lowerCAmelCase     ,							indent=2       )


def      SCREAMING_SNAKE_CASE      (   lowerCAmelCase     ,							lowerCAmelCase       ):
       os.makedirs(lowerCAmelCase     ,							exist_ok=lowerCAmelCase       )
       _UpperCamelCase														=    {}
       for name, parameter in state_dict.items():
              _UpperCamelCase														=    offload_weight(lowerCAmelCase     ,							lowerCAmelCase     ,							lowerCAmelCase     ,							index=lowerCAmelCase       )

       # Update index
       save_offload_index(lowerCAmelCase     ,							lowerCAmelCase       )

class 			__A(    __UpperCAmelCase			):

     def __init__(   self,							A,							A   ):





            """simple docstring"""

            _UpperCamelCase														=    dataset
            _UpperCamelCase														=    prefix

     def __getitem__(   self,							A   ):





            """simple docstring"""

            return self.dataset[F'''{self.prefix}{key}''']

     def __iter__(   self   ):





            """simple docstring"""

            return iter([key for key in self.dataset if key.startswith(self.prefix   )]   )



     def __len__(   self   ):





            """simple docstring"""

            return len(self.dataset   )






class 			__A(    __UpperCAmelCase			):

     def __init__(   self,							A = None,							A = None,							A = None,							A=None,							):





            """simple docstring"""

            if state_dict is None and save_folder is None:
                   raise ValueError('''Need either a `state_dict` or a `save_folder` containing offloaded weights.'''   )

            _UpperCamelCase														=    {} if state_dict is None else state_dict
            _UpperCamelCase														=    save_folder
            if index is None and save_folder is not None:
                   with open(os.path.join(A,							'''index.json'''   )   ) as f:
                          _UpperCamelCase														=    json.load(A   )
            _UpperCamelCase														=    {} if index is None else index
            _UpperCamelCase														=    list(self.state_dict.keys()   )
            self.all_keys.extend([key for key in self.index if key not in self.all_keys]   )
            _UpperCamelCase														=    device

     def __getitem__(   self,							A   ):





            """simple docstring"""

            if key in self.state_dict:
                   return self.state_dict[key]
            _UpperCamelCase														=    self.index[key]
            if weight_info.get('''safetensors_file'''   ) is not None:
                   if not is_safetensors_available():
                          raise ImportError('''These offloaded weights require the use of safetensors: `pip install safetensors`.'''   )

                   from safetensors import safe_open

                   _UpperCamelCase														=    '''cpu''' if self.device is None else self.device
                   with safe_open(weight_info['''safetensors_file'''],							framework='''pt''',							device=A   ) as f:
                          _UpperCamelCase														=    f.get_tensor(weight_info.get('''weight_name''',							A   )   )

                   if "dtype" in weight_info:
                          return tensor.to(getattr(A,							weight_info['''dtype''']   )   )
                   else:
                          return tensor

            _UpperCamelCase														=    os.path.join(self.save_folder,							F'''{key}.dat'''   )
            return load_offloaded_weight(A,							A   )

     def __iter__(   self   ):





            """simple docstring"""

            return iter(self.all_keys   )



     def __len__(   self   ):





            """simple docstring"""

            return len(self.all_keys   )






def      SCREAMING_SNAKE_CASE      (   lowerCAmelCase     ,							lowerCAmelCase       ):
       _UpperCamelCase														=    {}
       for module_name in submodule_names:
              # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
              # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
              result.update(
                  {
                      key: param
                      for key, param in state_dict.items()
                      if key == module_name or key.startswith(module_name + '''.'''       )
                  }       )
       return result






