




'''simple docstring'''





import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
  import torch

  from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
  import pyanvml.pyanvml as nvml


a		:	int       =      logging.get_logger(__name__)




class      UpperCamelCase       (	snake_case__       ):
  a     :PyTorchBenchmarkArguments
  a     :PretrainedConfig
  a     :str          =					"PyTorch"



  @property
  def     a_							(			self ):







      '''simple docstring'''


      return torch.__version__



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							Optional[int]							       =      self._prepare_inference_func(__lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase )
      return self._measure_speed(_inference )



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							List[str]							       =      self._prepare_inference_func(__lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase )
      return self._measure_memory(_inference )



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							List[Any]							       =      self._prepare_train_func(__lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase )
      return self._measure_speed(_train )



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							Any							       =      self._prepare_train_func(__lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase )
      return self._measure_memory(_train )



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							Tuple							       =      self.config_dict[model_name]

      if self.args.torchscript:
          _a				:							Any							       =      True

      _a				:							Any							       =      (
          hasattr(__lowerCAmelCase		,     """architectures""" )
          and isinstance(config.architectures		,     __lowerCAmelCase )
          and len(config.architectures ) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
          try:
              _a				:							Optional[int]							       =      config.architectures[0]
              _a				:							str							       =      __import__("""transformers"""		,     fromlist=[model_class] )
              _a				:							str							       =      getattr(__lowerCAmelCase		,     __lowerCAmelCase )
              _a				:							List[Any]							       =      model_cls(__lowerCAmelCase )
          except ImportError:
              raise ImportError(
                  f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                  """ set `--only_pretrain_model` or `args.only_pretrain_model=True`.""" )
      else:
          _a				:							List[str]							       =      MODEL_MAPPING[config.__class__](__lowerCAmelCase )

      model.eval()
      model.to(self.args.device )

      # encoder-decoder has vocab size saved differently
      _a				:							str							       =      config.vocab_size if hasattr(__lowerCAmelCase		,     """vocab_size""" ) else config.encoder.vocab_size
      _a				:							int							       =      torch.randint(__lowerCAmelCase		,     (batch_size, sequence_length)		,     dtype=torch.long		,     device=self.args.device )

      if self.args.fpaa:
          logger.info("""Running training in Mixed Precision...""" )
          if not self.args.is_gpu:
              raise ValueError("""Mixed precision is possible only for GPU.""" )
          # amp seems to have memory leaks so that memory usage
          # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
          model.half()

      if self.args.torchscript:
          with torch.no_grad():
              _a				:							str							       =      torch.jit.trace(__lowerCAmelCase		,     __lowerCAmelCase )
      else:
          _a				:							Tuple							       =      model

      def encoder_decoder_forward():
          with torch.no_grad():
              _a				:							Tuple							       =      inference_model(__lowerCAmelCase		,     decoder_input_ids=__lowerCAmelCase )
          return outputs

      def encoder_forward():
          with torch.no_grad():
              _a				:							Union[str, Any]							       =      inference_model(__lowerCAmelCase )
          return outputs

      _a				:							Union[str, Any]							       =      encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
      return _forward



  def     a_							(			self		,     __lowerCAmelCase		,     __lowerCAmelCase		,     __lowerCAmelCase ):







      '''simple docstring'''


      _a				:							int							       =      self.config_dict[model_name]

      _a				:							List[str]							       =      (
          hasattr(__lowerCAmelCase		,     """architectures""" )
          and isinstance(config.architectures		,     __lowerCAmelCase )
          and len(config.architectures ) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
          try:
              _a				:							Dict							       =      config.architectures[0]
              _a				:							Optional[Any]							       =      __import__("""transformers"""		,     fromlist=[model_class] )
              _a				:							int							       =      getattr(__lowerCAmelCase		,     __lowerCAmelCase )
              _a				:							List[str]							       =      model_cls(__lowerCAmelCase )
          except ImportError:
              raise ImportError(
                  f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                  """ set `--only_pretrain_model` or `args.only_pretrain_model=True`.""" )
      else:
          _a				:							Any							       =      MODEL_WITH_LM_HEAD_MAPPING[config.__class__](__lowerCAmelCase )

      if self.args.torchscript:
          raise NotImplementedError("""Training for torchscript is currently not implemented""" )
      else:
          _a				:							Tuple							       =      model

      model.train()
      model.to(self.args.device )

      # encoder-decoder has vocab size saved differently
      _a				:							Dict							       =      config.vocab_size if hasattr(__lowerCAmelCase		,     """vocab_size""" ) else config.encoder.vocab_size
      _a				:							Tuple							       =      torch.randint(__lowerCAmelCase		,     (batch_size, sequence_length)		,     dtype=torch.long		,     device=self.args.device )

      if self.args.fpaa:
          logger.info("""Running training in Mixed Precision...""" )
          if not self.args.is_gpu:
              raise ValueError("""Mixed precision is possible only for GPU.""" )

          # amp seems to have memory leaks so that memory usage
          # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
          model.half()

      def compute_loss_and_backprob_encoder():
          _a				:							int							       =      train_model(__lowerCAmelCase		,     labels=__lowerCAmelCase )[0]
          loss.backward()
          return loss

      def compute_loss_and_backprob_encoder_decoder():
          _a				:							str							       =      train_model(__lowerCAmelCase		,     decoder_input_ids=__lowerCAmelCase		,     labels=__lowerCAmelCase )[0]
          loss.backward()
          return loss

      _a				:							Any							       =      (
          compute_loss_and_backprob_encoder_decoder
          if config.is_encoder_decoder
          else compute_loss_and_backprob_encoder
      )
      return _train



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      try:
          if self.args.is_tpu or self.args.torchscript:
              # run additional 10 times to stabilize compilation for tpu and torchscript
              logger.info("""Do inference on TPU or torchscript. Running model 5 times to stabilize compilation""" )
              timeit.repeat(
                  __lowerCAmelCase		,     repeat=1		,     number=5		,     )

          # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
          _a				:							List[str]							       =      timeit.repeat(
              __lowerCAmelCase		,     repeat=self.args.repeat		,     number=10		,     )

          if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
              import torch_xla.debug.metrics as met

              self.print_fn(met.metrics_report() )

          return min(__lowerCAmelCase ) / 10.0
      except RuntimeError as e:
          self.print_fn(f'''Doesn\'t fit on GPU. {e}''' )
          return "N/A"



  def     a_							(			self		,     __lowerCAmelCase ):







      '''simple docstring'''


      try:
          if self.args.trace_memory_line_by_line:
              _a				:							Any							       =      start_memory_tracing("""transformers""" )

          if self.args.is_tpu:
              # tpu
              raise NotImplementedError(
                  """Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with"""
                  """ `--no-memory` or `args.memory=False`""" )
          elif self.args.is_gpu:
              if not is_pyanvml_available():
                  logger.warning(
                      """py3nvml not installed, we won't log GPU memory usage. """
                      """Install py3nvml (pip install py3nvml) to log information about GPU.""" )
                  _a				:							Tuple							       =      """N/A"""
              else:
                  logger.info(
                      """Measuring total GPU usage on GPU device. Make sure to not have additional processes running"""
                      """ on the same GPU.""" )
                  # init nvml
                  nvml.nvmlInit()
                  func()
                  _a				:							Optional[Any]							       =      nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx )
                  _a				:							str							       =      nvml.nvmlDeviceGetMemoryInfo(__lowerCAmelCase )
                  _a				:							Any							       =      meminfo.used
                  _a				:							str							       =      Memory(__lowerCAmelCase )
                  # shutdown nvml
                  nvml.nvmlShutdown()
          else:
              # cpu
              _a				:							Tuple							       =      measure_peak_memory_cpu(__lowerCAmelCase )
              _a				:							str							       =      Memory(__lowerCAmelCase ) if isinstance(__lowerCAmelCase		,     __lowerCAmelCase ) else memory_bytes

          if self.args.trace_memory_line_by_line:
              _a				:							str							       =      stop_memory_tracing(__lowerCAmelCase )
          else:
              _a				:							List[str]							       =      None

          return memory, summary
      except RuntimeError as e:
          self.print_fn(f'''Doesn\'t fit on GPU. {e}''' )
          return "N/A", None










'''simple docstring'''

import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp






def 						_UpperCAmelCase     (	)      ->      Optional[Any]:





						"""simple docstring"""
						lowercase_   : List[str]   	=      ArgumentParser(
						    description=(
						        'PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes'
						    )				)

						# Optional arguments for the launch helper
						parser.add_argument('--num_cores'    ,      type=a    ,      default=1    ,      help='Number of TPU cores to use (1 or 8).'				)

						# positional
						parser.add_argument(
						    'training_script'    ,      type=a    ,      help=(
						        'The full path to the single TPU training '
						        'program/script to be launched in parallel, '
						        'followed by all the arguments for the '
						        'training script'
						    )    ,      )

						# rest from the training program
						parser.add_argument('training_script_args'    ,      nargs=a				)

						return parser.parse_args()






def 						_UpperCAmelCase     (	)      ->      str:





						"""simple docstring"""
						lowercase_   : Optional[int]   	=      parse_args()

						# Import training_script as a module.
						lowercase_   : str   	=      Path(args.training_script				)
						sys.path.append(str(script_fpath.parent.resolve()				)				)
						lowercase_   : int   	=      script_fpath.stem
						lowercase_   : List[Any]   	=      importlib.import_module(a				)

						# Patch sys.argv
						lowercase_   : List[str]   	=      [args.training_script] + args.training_script_args + ['--tpu_num_cores', str(args.num_cores				)]

						xmp.spawn(mod._mp_fn    ,      args=()    ,      nprocs=args.num_cores				)


if __name__ == "__main__":
						main()
