


import json
import os
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import numpy as np
import torch

from .imports import is_safetensors_available







def  _lowerCamelCase(			lowercase__     ,				lowercase__     ,				lowercase__     ,				lowercase__=None			)		->			int:

  '''simple docstring'''







  __lowercase=    None
  # Check the string instead of the dtype to be compatible with versions of PyTorch that don't have bfloat16.
  if str(weight.dtype			) == "torch.bfloat16":
    # Need to reinterpret the underlined data as int16 since NumPy does not handle bfloat16s.
    __lowercase=    weight.view(torch.intaa			)
    __lowercase=    'bfloat16'
  __lowercase=    weight.cpu().numpy()
  __lowercase=    os.path.join(lowercase__     ,				F'{weight_name}.dat'			)
  if index is not None:
    if dtype is None:
      __lowercase=    str(array.dtype			)
    __lowercase=    {'dtype': dtype, 'shape': list(array.shape			)}
  if array.ndim == 0:
    __lowercase=    array[None]
  __lowercase=    np.memmap(lowercase__     ,				dtype=array.dtype     ,				mode='w+'     ,				shape=array.shape			)
  __lowercase=    array[:]
  file_array.flush()
  return index







def  _lowerCamelCase(			lowercase__     ,				lowercase__			)		->			List[str]:

  '''simple docstring'''







  __lowercase=    tuple(weight_info['shape']			)
  if shape == ():
    # NumPy memory-mapped arrays can't have 0 dims so it was saved as 1d tensor
    __lowercase=    (1,)

  __lowercase=    weight_info['dtype']
  if dtype == "bfloat16":
    # NumPy does not support bfloat16 so this was saved as a int16
    __lowercase=    'int16'

  __lowercase=    np.memmap(lowercase__     ,				dtype=lowercase__     ,				shape=lowercase__     ,				mode='r'			)

  if len(weight_info['shape']			) == 0:
    __lowercase=    weight[0]
  __lowercase=    torch.tensor(lowercase__			)
  if weight_info["dtype"] == "bfloat16":
    __lowercase=    weight.view(torch.bfloataa			)

  return weight







def  _lowerCamelCase(			lowercase__     ,				lowercase__			)		->			Optional[Any]:

  '''simple docstring'''







  if index is None or len(lowercase__			) == 0:
    # Nothing to save
    return

  __lowercase=    os.path.join(lowercase__     ,				'index.json'			)
  if os.path.isfile(lowercase__			):
    with open(lowercase__     ,				'r'     ,				encoding='utf-8'			) as f:
      __lowercase=    json.load(lowercase__			)
  else:
    __lowercase=    {}
  current_index.update(lowercase__			)

  with open(lowercase__     ,				'w'     ,				encoding='utf-8'			) as f:
    json.dump(lowercase__     ,				lowercase__     ,				indent=2			)







def  _lowerCamelCase(			lowercase__     ,				lowercase__			)		->			int:

  '''simple docstring'''







  os.makedirs(lowercase__     ,				exist_ok=lowercase__			)
  __lowercase=    {}
  for name, parameter in state_dict.items():
    __lowercase=    offload_weight(lowercase__     ,				lowercase__     ,				lowercase__     ,				index=lowercase__			)

  # Update index
  save_offload_index(lowercase__     ,				lowercase__			)





class    A					(     A_				):


       def __init__(self   ,       lowerCAmelCase   ,       lowerCAmelCase      ):
         __lowercase=    dataset
         __lowercase=    prefix


       def __getitem__(self   ,       lowerCAmelCase      ):
         return self.dataset[f'{self.prefix}{key}']


       def __iter__(self      ):
         return iter([key for key in self.dataset if key.startswith(self.prefix      )]      )







       def __len__(self      ):
         return len(self.dataset      )





class    A					(     A_				):


       def __init__(self   ,       lowerCAmelCase = None   ,       lowerCAmelCase = None   ,       lowerCAmelCase = None   ,       lowerCAmelCase=None   ,       ):
         if state_dict is None and save_folder is None:
           raise ValueError('Need either a `state_dict` or a `save_folder` containing offloaded weights.'      )

         __lowercase=    {} if state_dict is None else state_dict
         __lowercase=    save_folder
         if index is None and save_folder is not None:
           with open(os.path.join(lowerCAmelCase   ,       'index.json'      )      ) as f:
             __lowercase=    json.load(lowerCAmelCase      )
         __lowercase=    {} if index is None else index
         __lowercase=    list(self.state_dict.keys()      )
         self.all_keys.extend([key for key in self.index if key not in self.all_keys]      )
         __lowercase=    device


       def __getitem__(self   ,       lowerCAmelCase      ):
         # State dict gets priority
         if key in self.state_dict:
           return self.state_dict[key]
         __lowercase=    self.index[key]
         if weight_info.get('safetensors_file'      ) is not None:
           if not is_safetensors_available():
             raise ImportError('These offloaded weights require the use of safetensors: `pip install safetensors`.'      )

           from safetensors import safe_open

           __lowercase=    'cpu' if self.device is None else self.device
           with safe_open(weight_info['safetensors_file']   ,       framework='pt'   ,       device=lowerCAmelCase      ) as f:
             __lowercase=    f.get_tensor(weight_info.get('weight_name'   ,       lowerCAmelCase      )      )

           if "dtype" in weight_info:
             return tensor.to(getattr(lowerCAmelCase   ,       weight_info['dtype']      )      )
           else:
             return tensor

         __lowercase=    os.path.join(self.save_folder   ,       f'{key}.dat'      )
         return load_offloaded_weight(lowerCAmelCase   ,       lowerCAmelCase      )


       def __iter__(self      ):
         return iter(self.all_keys      )







       def __len__(self      ):
         return len(self.all_keys      )







def  _lowerCamelCase(			lowercase__     ,				lowercase__			)		->			str:

  '''simple docstring'''







  __lowercase=    {}
  for module_name in submodule_names:
    # We want to catch module_name parameter (module_name.xxx) or potentially module_name, but not any of the
    # submodules that could being like module_name (transformers.h.1 and transformers.h.10 for instance)
    result.update(
        {
            key: param
            for key, param in state_dict.items()
            if key == module_name or key.startswith(module_name + '.'			)
        }			)
  return result










from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase					 =	logging.get_logger(__name__)





class    A					(     A_				):
							UpperCamelCase_     :							Any =['''input_ids''', '''attention_mask''']


							def __init__(self   ,       lowerCAmelCase="[PAD]"   ,       lowerCAmelCase="[BOS]"   ,       lowerCAmelCase="[EOS]"   ,       lowerCAmelCase="[MASK]"   ,       lowerCAmelCase="[CLS]"   ,       lowerCAmelCase="[SEP]"   ,       lowerCAmelCase=2_0_4_8   ,       **lowerCAmelCase   ,       ):
									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else pad_token
									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else bos_token
									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else eos_token
									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else mask_token
									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else cls_token
									__lowercase=    AddedToken(lowerCAmelCase   ,       lstrip=lowerCAmelCase   ,       rstrip=lowerCAmelCase      ) if isinstance(lowerCAmelCase   ,       lowerCAmelCase      ) else sep_token

									super().__init__(
									    pad_token=lowerCAmelCase   ,       bos_token=lowerCAmelCase   ,       eos_token=lowerCAmelCase   ,       mask_token=lowerCAmelCase   ,       cls_token=lowerCAmelCase   ,       sep_token=lowerCAmelCase   ,       model_max_length=lowerCAmelCase   ,       **lowerCAmelCase   ,       )

									__lowercase=    2**8  # utf is 8 bits

									# define special tokens dict
									__lowercase=    {
									    self.pad_token: 0,
									    self.bos_token: 1,
									    self.eos_token: 2,
									    self.mask_token: 3,
									    self.cls_token: 4,
									    self.sep_token: 5,
									}
									__lowercase=    len(self.special_tokens_encoder      )
									__lowercase=    {v: k for k, v in self.special_tokens_encoder.items()}


							def     _A (self      ):
									__lowercase=    self.special_tokens_encoder.copy()
									vocab.update(self.added_tokens_encoder      )
									for i in range(self._utf_vocab_size      ):
											__lowercase=    chr(lowerCAmelCase      )
											__lowercase=    i + len(self.special_tokens_encoder      )
									return vocab


							@property
							def     _A (self      ):
									return self._utf_vocab_size + self._num_special_tokens


							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase = None   ,       lowerCAmelCase = False      ):
									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowerCAmelCase   ,       token_ids_a=lowerCAmelCase   ,       already_has_special_tokens=lowerCAmelCase      )

									# normal case: some special tokens
									if token_ids_a is None:
											return [1] + [0] * len(lowerCAmelCase      ) + [1]
									return [1] + ([0] * len(lowerCAmelCase      )) + [1] + ([0] * len(lowerCAmelCase      )) + [1]


							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase = None      ):
									if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									else:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]


							def     _A (self   ,       lowerCAmelCase      ):
									__lowercase=    [chr(lowerCAmelCase      ) for i in text.encode('utf-8'      )]
									return tokens


							def     _A (self   ,       lowerCAmelCase      ):
									if token in self.special_tokens_encoder:
											__lowercase=    self.special_tokens_encoder[token]
									elif token in self.added_tokens_encoder:
											__lowercase=    self.added_tokens_encoder[token]
									elif len(lowerCAmelCase      ) != 1:
											__lowercase=    self.unk_token_id
									else:
											__lowercase=    ord(lowerCAmelCase      ) + self._num_special_tokens
									return token_id


							def     _A (self   ,       lowerCAmelCase      ):
									if index in self.special_tokens_decoder:
											__lowercase=    self.special_tokens_decoder[index]
									elif index in self.added_tokens_decoder:
											__lowercase=    self.added_tokens_decoder[index]
									else:
											__lowercase=    chr(index - self._num_special_tokens      )
									return token


							def     _A (self   ,       lowerCAmelCase      ):
									__lowercase=    B''
									for token in tokens:
											if token in self.special_tokens_decoder:
													__lowercase=    self.special_tokens_decoder[token].encode('utf-8'      )
											elif token in self.added_tokens_decoder:
													__lowercase=    self.special_tokens_decoder[token].encode('utf-8'      )
											elif token in self.special_tokens_encoder:
													__lowercase=    token.encode('utf-8'      )
											elif token in self.added_tokens_encoder:
													__lowercase=    token.encode('utf-8'      )
											else:
													__lowercase=    bytes([ord(lowerCAmelCase      )]      )
											bstring += tok_string
									__lowercase=    bstring.decode('utf-8'   ,       errors='replace'      )
									return string







							def     _A (self   ,       lowerCAmelCase   ,       lowerCAmelCase = None      ):
									return ()






