
import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
 import torch

 from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
 import pyanvml.pyanvml as nvml


lowerCamelCase_				:							Dict       =    logging.get_logger(__name__)







class  _lowerCamelCase			(lowerCamelCase					):
     lowercase__				  =			42
     lowercase__				  =			42
     lowercase__				  =			"PyTorch"

     @property
     def 						__lowerCamelCase				(  self    ):
            return torch.__version__

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   self._prepare_inference_func(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )
            return self._measure_speed(_inference    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   self._prepare_inference_func(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )
            return self._measure_memory(_inference    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   self._prepare_train_func(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )
            return self._measure_speed(_train    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   self._prepare_train_func(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )
            return self._measure_memory(_train    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   self.config_dict[model_name]

            if self.args.torchscript:
                   __snake_case						       =   True

            __snake_case						       =   (
                hasattr(SCREAMING_SNAKE_CASE_							,       'architectures'    )
                and isinstance(config.architectures							,       SCREAMING_SNAKE_CASE_    )
                and len(config.architectures    ) > 0
            )
            if not self.args.only_pretrain_model and has_model_class_in_config:
                   try:
                          __snake_case						       =   config.architectures[0]
                          __snake_case						       =   __import__('transformers'							,       fromlist=[model_class]    )
                          __snake_case						       =   getattr(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )
                          __snake_case						       =   model_cls(SCREAMING_SNAKE_CASE_    )
                   except ImportError:
                          raise ImportError(
                              f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                              ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'    )
            else:
                   __snake_case						       =   MODEL_MAPPING[config.__class__](SCREAMING_SNAKE_CASE_    )

            model.eval()
            model.to(self.args.device    )

            # encoder-decoder has vocab size saved differently
            __snake_case						       =   config.vocab_size if hasattr(SCREAMING_SNAKE_CASE_							,       'vocab_size'    ) else config.encoder.vocab_size
            __snake_case						       =   torch.randint(SCREAMING_SNAKE_CASE_							,       (batch_size, sequence_length)							,       dtype=torch.long							,       device=self.args.device    )

            if self.args.fpaa:
                   logger.info('Running training in Mixed Precision...'    )
                   if not self.args.is_gpu:
                          raise ValueError('Mixed precision is possible only for GPU.'    )
                   # amp seems to have memory leaks so that memory usage
                   # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
                   model.half()

            if self.args.torchscript:
                   with torch.no_grad():
                          __snake_case						       =   torch.jit.trace(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )
            else:
                   __snake_case						       =   model

            def encoder_decoder_forward():
                   with torch.no_grad():
                          __snake_case						       =   inference_model(SCREAMING_SNAKE_CASE_							,       decoder_input_ids=SCREAMING_SNAKE_CASE_    )
                   return outputs

            def encoder_forward():
                   with torch.no_grad():
                          __snake_case						       =   inference_model(SCREAMING_SNAKE_CASE_    )
                   return outputs

            __snake_case						       =   encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
            return _forward

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   self.config_dict[model_name]

            __snake_case						       =   (
                hasattr(SCREAMING_SNAKE_CASE_							,       'architectures'    )
                and isinstance(config.architectures							,       SCREAMING_SNAKE_CASE_    )
                and len(config.architectures    ) > 0
            )
            if not self.args.only_pretrain_model and has_model_class_in_config:
                   try:
                          __snake_case						       =   config.architectures[0]
                          __snake_case						       =   __import__('transformers'							,       fromlist=[model_class]    )
                          __snake_case						       =   getattr(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )
                          __snake_case						       =   model_cls(SCREAMING_SNAKE_CASE_    )
                   except ImportError:
                          raise ImportError(
                              f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                              ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'    )
            else:
                   __snake_case						       =   MODEL_WITH_LM_HEAD_MAPPING[config.__class__](SCREAMING_SNAKE_CASE_    )

            if self.args.torchscript:
                   raise NotImplementedError('Training for torchscript is currently not implemented'    )
            else:
                   __snake_case						       =   model

            model.train()
            model.to(self.args.device    )

            # encoder-decoder has vocab size saved differently
            __snake_case						       =   config.vocab_size if hasattr(SCREAMING_SNAKE_CASE_							,       'vocab_size'    ) else config.encoder.vocab_size
            __snake_case						       =   torch.randint(SCREAMING_SNAKE_CASE_							,       (batch_size, sequence_length)							,       dtype=torch.long							,       device=self.args.device    )

            if self.args.fpaa:
                   logger.info('Running training in Mixed Precision...'    )
                   if not self.args.is_gpu:
                          raise ValueError('Mixed precision is possible only for GPU.'    )

                   # amp seems to have memory leaks so that memory usage
                   # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
                   model.half()

            def compute_loss_and_backprob_encoder():
                   __snake_case						       =   train_model(SCREAMING_SNAKE_CASE_							,       labels=SCREAMING_SNAKE_CASE_    )[0]
                   loss.backward()
                   return loss

            def compute_loss_and_backprob_encoder_decoder():
                   __snake_case						       =   train_model(SCREAMING_SNAKE_CASE_							,       decoder_input_ids=SCREAMING_SNAKE_CASE_							,       labels=SCREAMING_SNAKE_CASE_    )[0]
                   loss.backward()
                   return loss

            __snake_case						       =   (
                compute_loss_and_backprob_encoder_decoder
                if config.is_encoder_decoder
                else compute_loss_and_backprob_encoder
            )
            return _train

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_    ):
            try:
                   if self.args.is_tpu or self.args.torchscript:
                          # run additional 10 times to stabilize compilation for tpu and torchscript
                          logger.info('Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'    )
                          timeit.repeat(
                              SCREAMING_SNAKE_CASE_							,       repeat=1							,       number=5							,       )

                   # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                   __snake_case						       =   timeit.repeat(
                       SCREAMING_SNAKE_CASE_							,       repeat=self.args.repeat							,       number=10							,       )

                   if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
                          import torch_xla.debug.metrics as met

                          self.print_fn(met.metrics_report()    )

                   return min(SCREAMING_SNAKE_CASE_    ) / 1_0.0
            except RuntimeError as e:
                   self.print_fn(f'''Doesn\'t fit on GPU. {e}'''    )
                   return "N/A"

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_    ):
            try:
                   if self.args.trace_memory_line_by_line:
                          __snake_case						       =   start_memory_tracing('transformers'    )

                   if self.args.is_tpu:
                          # tpu
                          raise NotImplementedError(
                              'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'
                              ' `--no-memory` or `args.memory=False`'    )
                   elif self.args.is_gpu:
                          if not is_pyanvml_available():
                                 logger.warning(
                                     'py3nvml not installed, we won\'t log GPU memory usage. '
                                     'Install py3nvml (pip install py3nvml) to log information about GPU.'    )
                                 __snake_case						       =   'N/A'
                          else:
                                 logger.info(
                                     'Measuring total GPU usage on GPU device. Make sure to not have additional processes running'
                                     ' on the same GPU.'    )
                                 # init nvml
                                 nvml.nvmlInit()
                                 func()
                                 __snake_case						       =   nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
                                 __snake_case						       =   nvml.nvmlDeviceGetMemoryInfo(SCREAMING_SNAKE_CASE_    )
                                 __snake_case						       =   meminfo.used
                                 __snake_case						       =   Memory(SCREAMING_SNAKE_CASE_    )
                                 # shutdown nvml
                                 nvml.nvmlShutdown()
                   else:
                          # cpu
                          __snake_case						       =   measure_peak_memory_cpu(SCREAMING_SNAKE_CASE_    )
                          __snake_case						       =   Memory(SCREAMING_SNAKE_CASE_    ) if isinstance(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    ) else memory_bytes

                   if self.args.trace_memory_line_by_line:
                          __snake_case						       =   stop_memory_tracing(SCREAMING_SNAKE_CASE_    )
                   else:
                          __snake_case						       =   None

                   return memory, summary
            except RuntimeError as e:
                   self.print_fn(f'''Doesn\'t fit on GPU. {e}'''    )
                   return "N/A", None


import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class  _lowerCamelCase			(lowerCamelCase							,					unittest.TestCase					):
     lowercase__				  =			HerbertTokenizer
     lowercase__				  =			HerbertTokenizerFast
     lowercase__				  =			True

     def 						__lowerCamelCase				(  self    ):
            super().setUp()

            # Use a simpler test file without japanese/chinese characters
            with open(f'''{get_tests_dir()}/fixtures/sample_text_no_unicode.txt'''							,       encoding='utf-8'    ) as f_data:
                   __snake_case						       =   f_data.read().replace('\n\n'							,       '\n'    ).strip()

            __snake_case						       =   [
                '<s>',
                '</s>',
                'l',
                'o',
                'w',
                'e',
                'r',
                's',
                't',
                'i',
                'd',
                'n',
                'w</w>',
                'r</w>',
                't</w>',
                'lo',
                'low',
                'er</w>',
                'low</w>',
                'lowest</w>',
                'newer</w>',
                'wider</w>',
                ',</w>',
                '<unk>',
            ]
            __snake_case						       =   dict(zip(SCREAMING_SNAKE_CASE_							,       range(len(SCREAMING_SNAKE_CASE_    )    )    )    )
            __snake_case						       =   ['l o 123', 'lo w 1456', 'e r</w> 1789', '']

            __snake_case						       =   os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES['vocab_file']    )
            __snake_case						       =   os.path.join(self.tmpdirname							,       VOCAB_FILES_NAMES['merges_file']    )
            with open(self.vocab_file							,       'w'    ) as fp:
                   fp.write(json.dumps(SCREAMING_SNAKE_CASE_    )    )
            with open(self.merges_file							,       'w'    ) as fp:
                   fp.write('\n'.join(SCREAMING_SNAKE_CASE_    )    )

     def 						__lowerCamelCase				(  self							,       SCREAMING_SNAKE_CASE_    ):
            __snake_case						       =   'lower newer'
            __snake_case						       =   'lower newer'
            return input_text, output_text

     def 						__lowerCamelCase				(  self    ):
            __snake_case						       =   self.tokenizer_class(vocab_file=self.vocab_file							,       merges_file=self.merges_file    )

            __snake_case						       =   'lower'
            __snake_case						       =   ['low', 'er</w>']
            __snake_case						       =   tokenizer.tokenize(SCREAMING_SNAKE_CASE_    )
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )

            __snake_case						       =   tokens + ['<unk>']
            __snake_case						       =   [16, 17, 23]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_    )							,       SCREAMING_SNAKE_CASE_    )

     def 						__lowerCamelCase				(  self    ):
            if not self.test_rust_tokenizer:
                   return

            __snake_case						       =   self.get_tokenizer()
            __snake_case						       =   self.get_rust_tokenizer()

            __snake_case						       =   'lower,newer'

            __snake_case						       =   tokenizer.tokenize(SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_    )
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )

            __snake_case						       =   tokenizer.encode(SCREAMING_SNAKE_CASE_							,       add_special_tokens=SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   rust_tokenizer.encode(SCREAMING_SNAKE_CASE_							,       add_special_tokens=SCREAMING_SNAKE_CASE_    )
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )

            __snake_case						       =   self.get_rust_tokenizer()
            __snake_case						       =   tokenizer.encode(SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   rust_tokenizer.encode(SCREAMING_SNAKE_CASE_    )
            self.assertListEqual(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )

     @slow
     def 						__lowerCamelCase				(  self    ):
            __snake_case						       =   self.tokenizer_class.from_pretrained('allegro/herbert-base-cased'    )

            __snake_case						       =   tokenizer.encode('konstruowanie sekwencji'							,       add_special_tokens=SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   tokenizer.encode('konstruowanie wielu sekwencji'							,       add_special_tokens=SCREAMING_SNAKE_CASE_    )

            __snake_case						       =   tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_    )
            __snake_case						       =   tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_    )

            assert encoded_sentence == [0] + text + [2]
            assert encoded_pair == [0] + text + [2] + text_a + [2]
