import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
     import torch

     from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
     import pyanvml.pyanvml as nvml


__snake_case         =     logging.get_logger(__name__)







class       __snake_case (  snake_case__ ):
 __lowerCamelCase     : PyTorchBenchmarkArguments
 __lowerCamelCase     : PretrainedConfig
 __lowerCamelCase     : str 	=   "PyTorch"
 @property
 def 						UpperCAmelCase__							(	self    )						->				Union[str, Any]:
       '''simple docstring'''





       return torch.__version__
 def 						UpperCAmelCase__							(	self		, snake_case__		, snake_case__		, snake_case__    )						->				Tuple:
       '''simple docstring'''





       UpperCAmelCase     :			Optional[Any]     	=self._prepare_inference_func(__snake_case		, __snake_case		, __snake_case    )
       return self._measure_speed(_inference    )
 def 						UpperCAmelCase__							(	self		, snake_case__		, snake_case__		, snake_case__    )						->				Optional[Any]:
       '''simple docstring'''





       UpperCAmelCase     :			List[str]     	=self._prepare_inference_func(__snake_case		, __snake_case		, __snake_case    )
       return self._measure_memory(_inference    )
 def 						UpperCAmelCase__							(	self		, snake_case__		, snake_case__		, snake_case__    )						->				Any:
       '''simple docstring'''





       UpperCAmelCase     :			Union[str, Any]     	=self._prepare_train_func(__snake_case		, __snake_case		, __snake_case    )
       return self._measure_speed(_train    )
 def 						UpperCAmelCase__							(	self		, snake_case__		, snake_case__		, snake_case__    )						->				Union[str, Any]:
       '''simple docstring'''





       UpperCAmelCase     :			Optional[int]     	=self._prepare_train_func(__snake_case		, __snake_case		, __snake_case    )
       return self._measure_memory(_train    )
 def 						UpperCAmelCase__							(	self		, snake_case__		, snake_case__		, snake_case__    )						->				str:
       '''simple docstring'''





       UpperCAmelCase     :			int     	=self.config_dict[model_name]

       if self.args.torchscript:
             UpperCAmelCase     :			Any     	=True

       UpperCAmelCase     :			str     	=(
           hasattr(__snake_case		, '''architectures'''    )
           and isinstance(config.architectures		, __snake_case    )
           and len(config.architectures    ) > 0
       )
       if not self.args.only_pretrain_model and has_model_class_in_config:
             try:
                   UpperCAmelCase     :			List[Any]     	=config.architectures[0]
                   UpperCAmelCase     :			Any     	=__import__('''transformers'''		, fromlist=[model_class]    )
                   UpperCAmelCase     :			int     	=getattr(__snake_case		, __snake_case    )
                   UpperCAmelCase     :			List[Any]     	=model_cls(__snake_case    )
             except ImportError:
                   raise ImportError(
                       f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                       ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
       else:
             UpperCAmelCase     :			Union[str, Any]     	=MODEL_MAPPING[config.__class__](__snake_case    )

       model.eval()
       model.to(self.args.device    )

       # encoder-decoder has vocab size saved differently
       UpperCAmelCase     :			Tuple     	=config.vocab_size if hasattr(__snake_case		, '''vocab_size'''    ) else config.encoder.vocab_size
       UpperCAmelCase     :			Optional[Any]     	=torch.randint(__snake_case		, (batch_size, sequence_length)		, dtype=torch.long		, device=self.args.device    )

       if self.args.fpaa:
             logger.info('''Running training in Mixed Precision...'''    )
             if not self.args.is_gpu:
                   raise ValueError('''Mixed precision is possible only for GPU.'''    )
             # amp seems to have memory leaks so that memory usage
             # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
             model.half()

       if self.args.torchscript:
             with torch.no_grad():
                   UpperCAmelCase     :			List[Any]     	=torch.jit.trace(__snake_case		, __snake_case    )
       else:
             UpperCAmelCase     :			Dict     	=model

       def encoder_decoder_forward():
             with torch.no_grad():
                   UpperCAmelCase     :			Dict     	=inference_model(__snake_case		, decoder_input_ids=__snake_case    )
             return outputs

       def encoder_forward():
             with torch.no_grad():
                   UpperCAmelCase     :			Tuple     	=inference_model(__snake_case    )
             return outputs

       UpperCAmelCase     :			Optional[int]     	=encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
       return _forward
 def 						UpperCAmelCase__							(	self		, snake_case__		, snake_case__		, snake_case__    )						->				Any:
       '''simple docstring'''





       UpperCAmelCase     :			Optional[int]     	=self.config_dict[model_name]

       UpperCAmelCase     :			str     	=(
           hasattr(__snake_case		, '''architectures'''    )
           and isinstance(config.architectures		, __snake_case    )
           and len(config.architectures    ) > 0
       )
       if not self.args.only_pretrain_model and has_model_class_in_config:
             try:
                   UpperCAmelCase     :			List[str]     	=config.architectures[0]
                   UpperCAmelCase     :			Optional[int]     	=__import__('''transformers'''		, fromlist=[model_class]    )
                   UpperCAmelCase     :			List[str]     	=getattr(__snake_case		, __snake_case    )
                   UpperCAmelCase     :			Tuple     	=model_cls(__snake_case    )
             except ImportError:
                   raise ImportError(
                       f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                       ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
       else:
             UpperCAmelCase     :			Union[str, Any]     	=MODEL_WITH_LM_HEAD_MAPPING[config.__class__](__snake_case    )

       if self.args.torchscript:
             raise NotImplementedError('''Training for torchscript is currently not implemented'''    )
       else:
             UpperCAmelCase     :			Optional[Any]     	=model

       model.train()
       model.to(self.args.device    )

       # encoder-decoder has vocab size saved differently
       UpperCAmelCase     :			Dict     	=config.vocab_size if hasattr(__snake_case		, '''vocab_size'''    ) else config.encoder.vocab_size
       UpperCAmelCase     :			Dict     	=torch.randint(__snake_case		, (batch_size, sequence_length)		, dtype=torch.long		, device=self.args.device    )

       if self.args.fpaa:
             logger.info('''Running training in Mixed Precision...'''    )
             if not self.args.is_gpu:
                   raise ValueError('''Mixed precision is possible only for GPU.'''    )

             # amp seems to have memory leaks so that memory usage
             # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
             model.half()

       def compute_loss_and_backprob_encoder():
             UpperCAmelCase     :			str     	=train_model(__snake_case		, labels=__snake_case    )[0]
             loss.backward()
             return loss

       def compute_loss_and_backprob_encoder_decoder():
             UpperCAmelCase     :			Union[str, Any]     	=train_model(__snake_case		, decoder_input_ids=__snake_case		, labels=__snake_case    )[0]
             loss.backward()
             return loss

       UpperCAmelCase     :			int     	=(
           compute_loss_and_backprob_encoder_decoder
           if config.is_encoder_decoder
           else compute_loss_and_backprob_encoder
       )
       return _train
 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				Tuple:
       '''simple docstring'''





       try:
             if self.args.is_tpu or self.args.torchscript:
                   # run additional 10 times to stabilize compilation for tpu and torchscript
                   logger.info('''Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'''    )
                   timeit.repeat(
                       __snake_case		, repeat=1		, number=5		, )

             # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
             UpperCAmelCase     :			Dict     	=timeit.repeat(
                 __snake_case		, repeat=self.args.repeat		, number=10		, )

             if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
                   import torch_xla.debug.metrics as met

                   self.print_fn(met.metrics_report()    )

             return min(__snake_case    ) / 10.0
       except RuntimeError as e:
             self.print_fn(f'''Doesn\'t fit on GPU. {e}'''    )
             return "N/A"






 def 						UpperCAmelCase__							(	self		, snake_case__    )						->				List[Any]:
       '''simple docstring'''





       try:
             if self.args.trace_memory_line_by_line:
                   UpperCAmelCase     :			Dict     	=start_memory_tracing('''transformers'''    )

             if self.args.is_tpu:
                   # tpu
                   raise NotImplementedError(
                       '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'''
                       ''' `--no-memory` or `args.memory=False`'''    )
             elif self.args.is_gpu:
                   if not is_pyanvml_available():
                         logger.warning(
                             '''py3nvml not installed, we won\'t log GPU memory usage. '''
                             '''Install py3nvml (pip install py3nvml) to log information about GPU.'''    )
                         UpperCAmelCase     :			Optional[Any]     	='''N/A'''
                   else:
                         logger.info(
                             '''Measuring total GPU usage on GPU device. Make sure to not have additional processes running'''
                             ''' on the same GPU.'''    )
                         # init nvml
                         nvml.nvmlInit()
                         func()
                         UpperCAmelCase     :			int     	=nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
                         UpperCAmelCase     :			Optional[Any]     	=nvml.nvmlDeviceGetMemoryInfo(__snake_case    )
                         UpperCAmelCase     :			Dict     	=meminfo.used
                         UpperCAmelCase     :			Tuple     	=Memory(__snake_case    )
                         # shutdown nvml
                         nvml.nvmlShutdown()
             else:
                   # cpu
                   UpperCAmelCase     :			List[Any]     	=measure_peak_memory_cpu(__snake_case    )
                   UpperCAmelCase     :			str     	=Memory(__snake_case    ) if isinstance(__snake_case		, __snake_case    ) else memory_bytes

             if self.args.trace_memory_line_by_line:
                   UpperCAmelCase     :			str     	=stop_memory_tracing(__snake_case    )
             else:
                   UpperCAmelCase     :			Optional[Any]     	=None

             return memory, summary
       except RuntimeError as e:
             self.print_fn(f'''Doesn\'t fit on GPU. {e}'''    )
             return "N/A", None




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation

__UpperCamelCase					:	Dict          =				logging.get_logger(__name__)

__UpperCamelCase					:	List[str]          =				{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__UpperCamelCase					:	Optional[int]          =				{
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
}

__UpperCamelCase					:	Any          =				{
    'gpt2': 1024,
    'gpt2-medium': 1024,
    'gpt2-large': 1024,
    'gpt2-xl': 1024,
    'distilgpt2': 1024,
}
@lru_cache()
def   snake_case_    (			):
   UpperCAmelCase_				:   Union[str, Any]							= (
       list(range(ord('''!'''							)      ,							ord('''~'''							) + 1							)							) + list(range(ord('''¡'''							)      ,							ord('''¬'''							) + 1							)							) + list(range(ord('''®'''							)      ,							ord('''ÿ'''							) + 1							)							)
   )
   UpperCAmelCase_				:   Optional[int]							= bs[:]
   UpperCAmelCase_				:   Union[str, Any]							= 0
   for b in range(2**8							):
      if b not in bs:
         bs.append(__lowercase							)
         cs.append(2**8 + n							)
         n += 1
   UpperCAmelCase_				:   Dict							= [chr(__lowercase							) for n in cs]
   return dict(zip(__lowercase      ,							__lowercase							)							)
def   snake_case_    (			__lowercase							):
   UpperCAmelCase_				:   List[str]							= set()
   UpperCAmelCase_				:   Optional[Any]							= word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)							)
      UpperCAmelCase_				:   Union[str, Any]							= char
   return pairs





class 			lowerCAmelCase__(     snake_case__    ):






    '''simple docstring'''





    A_	:  Optional[int]								=			VOCAB_FILES_NAMES
    A_	:  Any								=			PRETRAINED_VOCAB_FILES_MAP
    A_	:  Dict								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    A_	:  List[str]								=			['input_ids', 'attention_mask']





    def __init__(						self					: Optional[int]							, __snake_case					: Dict							, __snake_case					: List[Any]							, __snake_case					: List[Any]="replace"							, __snake_case					: Any="<|endoftext|>"							, __snake_case					: Union[str, Any]="<|endoftext|>"							, __snake_case					: List[str]="<|endoftext|>"							, __snake_case					: Any=None							, __snake_case					: List[str]=False							, __snake_case					: Optional[int]=False							, **__snake_case					: str							, ):





       '''simple docstring'''
       UpperCAmelCase_				:   Optional[int]							= AddedToken(__snake_case							, lstrip=__snake_case							, rstrip=__snake_case						) if isinstance(__snake_case							, __snake_case						) else bos_token
       UpperCAmelCase_				:   List[Any]							= AddedToken(__snake_case							, lstrip=__snake_case							, rstrip=__snake_case						) if isinstance(__snake_case							, __snake_case						) else eos_token
       UpperCAmelCase_				:   Optional[int]							= AddedToken(__snake_case							, lstrip=__snake_case							, rstrip=__snake_case						) if isinstance(__snake_case							, __snake_case						) else unk_token
       UpperCAmelCase_				:   Tuple							= AddedToken(__snake_case							, lstrip=__snake_case							, rstrip=__snake_case						) if isinstance(__snake_case							, __snake_case						) else pad_token
       super().__init__(
           errors=__snake_case							, unk_token=__snake_case							, bos_token=__snake_case							, eos_token=__snake_case							, pad_token=__snake_case							, add_prefix_space=__snake_case							, add_bos_token=__snake_case							, **__snake_case							, )
       UpperCAmelCase_				:   List[str]							= add_bos_token

       with open(__snake_case							, encoding='''utf-8'''						) as vocab_handle:
          UpperCAmelCase_				:   Optional[int]							= json.load(__snake_case						)
       UpperCAmelCase_				:   int							= {v: k for k, v in self.encoder.items()}
       UpperCAmelCase_				:   Union[str, Any]							= errors  # how to handle errors in decoding
       UpperCAmelCase_				:   Optional[int]							= bytes_to_unicode()
       UpperCAmelCase_				:   str							= {v: k for k, v in self.byte_encoder.items()}
       with open(__snake_case							, encoding='''utf-8'''						) as merges_handle:
          UpperCAmelCase_				:   Any							= merges_handle.read().split('''\n'''						)[1:-1]
       UpperCAmelCase_				:   List[str]							= [tuple(merge.split()						) for merge in bpe_merges]
       UpperCAmelCase_				:   Optional[int]							= dict(zip(__snake_case							, range(len(__snake_case						)						)						)						)
       UpperCAmelCase_				:   Any							= {}
       UpperCAmelCase_				:   int							= add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       UpperCAmelCase_				:   int							= re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''						)





    @property
    def        _lowerCamelCase			(						self					: int						):





       '''simple docstring'''
       return len(self.encoder						)





    def        _lowerCamelCase			(						self					: str						):





       '''simple docstring'''
       return dict(self.encoder							, **self.added_tokens_encoder						)





    def        _lowerCamelCase			(						self					: Optional[Any]							, __snake_case					: Optional[Any]						):





       '''simple docstring'''
       if token in self.cache:
          return self.cache[token]
       UpperCAmelCase_				:   Any							= tuple(__snake_case						)
       UpperCAmelCase_				:   Any							= get_pairs(__snake_case						)

       if not pairs:
          return token

       while True:
          UpperCAmelCase_				:   Any							= min(__snake_case							, key=lambda __snake_case						: self.bpe_ranks.get(__snake_case							, float('''inf'''						)						)						)
          if bigram not in self.bpe_ranks:
             break
          UpperCAmelCase_		,   UpperCAmelCase_				:   Dict							= bigram
          UpperCAmelCase_				:   List[Any]							= []
          UpperCAmelCase_				:   Union[str, Any]							= 0
          while i < len(__snake_case						):
             try:
                UpperCAmelCase_				:   Optional[int]							= word.index(__snake_case							, __snake_case						)
             except ValueError:
                new_word.extend(word[i:]						)
                break
             else:
                new_word.extend(word[i:j]						)
                UpperCAmelCase_				:   str							= j

             if word[i] == first and i < len(__snake_case						) - 1 and word[i + 1] == second:
                new_word.append(first + second						)
                i += 2
             else:
                new_word.append(word[i]						)
                i += 1
          UpperCAmelCase_				:   Any							= tuple(__snake_case						)
          UpperCAmelCase_				:   Tuple							= new_word
          if len(__snake_case						) == 1:
             break
          else:
             UpperCAmelCase_				:   str							= get_pairs(__snake_case						)
       UpperCAmelCase_				:   Union[str, Any]							= ''' '''.join(__snake_case						)
       UpperCAmelCase_				:   Optional[Any]							= word
       return word





    def        _lowerCamelCase			(						self					: List[Any]							, __snake_case					: Any							, __snake_case					: List[str]=None						):





       '''simple docstring'''
       if self.add_bos_token:
          UpperCAmelCase_				:   List[Any]							= [self.bos_token_id]
       else:
          UpperCAmelCase_				:   Dict							= []

       UpperCAmelCase_				:   Dict							= bos_token_ids + token_ids_a

       if token_ids_a is None:
          return output

       return output + bos_token_ids + token_ids_a





    def        _lowerCamelCase			(						self					: Optional[int]							, __snake_case					: List[int]							, __snake_case					: Optional[List[int]] = None							, __snake_case					: bool = False						):





       '''simple docstring'''
       if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=__snake_case							, token_ids_a=__snake_case							, already_has_special_tokens=__snake_case						)

       if not self.add_bos_token:
          return super().get_special_tokens_mask(
              token_ids_a=__snake_case							, token_ids_a=__snake_case							, already_has_special_tokens=__snake_case						)

       if token_ids_a is None:
          return [1] + ([0] * len(__snake_case						))
       return [1] + ([0] * len(__snake_case						)) + [1] + ([0] * len(__snake_case						))





    def        _lowerCamelCase			(						self					: str							, __snake_case					: Union[str, Any]						):





       '''simple docstring'''
       UpperCAmelCase_				:   List[str]							= []
       for token in re.findall(self.pat							, __snake_case						):
          UpperCAmelCase_				:   Any							= ''''''.join(
              self.byte_encoder[b] for b in token.encode('''utf-8'''						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
          bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__snake_case						).split(''' '''						)						)
       return bpe_tokens





    def        _lowerCamelCase			(						self					: Optional[Any]							, __snake_case					: Optional[int]						):





       '''simple docstring'''
       return self.encoder.get(__snake_case							, self.encoder.get(self.unk_token						)						)





    def        _lowerCamelCase			(						self					: int							, __snake_case					: str						):





       '''simple docstring'''
       return self.decoder.get(__snake_case						)





    def        _lowerCamelCase			(						self					: Union[str, Any]							, __snake_case					: List[str]						):





       '''simple docstring'''
       UpperCAmelCase_				:   List[Any]							= ''''''.join(__snake_case						)
       UpperCAmelCase_				:   str							= bytearray([self.byte_decoder[c] for c in text]						).decode('''utf-8'''							, errors=self.errors						)
       return text





    def        _lowerCamelCase			(						self					: str							, __snake_case					: str							, __snake_case					: Optional[str] = None						):





       '''simple docstring'''
       if not os.path.isdir(__snake_case						):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
          return
       UpperCAmelCase_				:   List[str]							= os.path.join(
           __snake_case							, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
       UpperCAmelCase_				:   List[Any]							= os.path.join(
           __snake_case							, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

       with open(__snake_case							, '''w'''							, encoding='''utf-8'''						) as f:
          f.write(json.dumps(self.encoder							, indent=2							, sort_keys=__snake_case							, ensure_ascii=__snake_case						) + '''\n'''						)

       UpperCAmelCase_				:   Union[str, Any]							= 0
       with open(__snake_case							, '''w'''							, encoding='''utf-8'''						) as writer:
          writer.write('''#version: 0.2\n'''						)
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							, key=lambda __snake_case						: kv[1]						):
             if index != token_index:
                logger.warning(
                    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                    ''' Please check that the tokenizer is not corrupted!'''						)
                UpperCAmelCase_				:   Dict							= token_index
             writer.write(''' '''.join(__snake_case						) + '''\n'''						)
             index += 1

       return vocab_file, merge_file





    def        _lowerCamelCase			(						self					: List[Any]							, __snake_case					: str							, __snake_case					: List[Any]=False							, **__snake_case					: Dict						):





       '''simple docstring'''
       UpperCAmelCase_				:   List[Any]							= kwargs.pop('''add_prefix_space'''							, self.add_prefix_space						)
       if is_split_into_words or add_prefix_space:
          UpperCAmelCase_				:   str							= ''' ''' + text
       return (text, kwargs)





    def        _lowerCamelCase			(						self					: Union[str, Any]							, __snake_case					: "Conversation"						):





       '''simple docstring'''
       UpperCAmelCase_				:   List[Any]							= []
       for is_user, text in conversation.iter_texts():
          input_ids.extend(self.encode(__snake_case							, add_special_tokens=__snake_case						) + [self.eos_token_id]						)
       if len(__snake_case						) > self.model_max_length:
          UpperCAmelCase_				:   str							= input_ids[-self.model_max_length :]
       return input_ids