

import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
  import torch

_a									=  logging.get_logger(__name__)







def _a					(			)       ->       Union[str, Any]:

 """simple docstring"""






 return importlib.util.find_spec('deepspeed'						) is not None


if is_accelerate_available() and is_deepspeed_available():
  from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
  # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
  # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
  from builtins import object as DeepSpeedConfig






class 				A_					(						snake_case__   ):


     def __init__(				self     :						str     ,    UpperCAmelCase     :						int   )				->      Optional[Any]:
      # set global weakref object
      set_hf_deepspeed_config(self   )
      dep_version_check('accelerate'   )
      dep_version_check('deepspeed'   )
      super().__init__(UpperCAmelCase   )







class 				A_					(						snake_case__   ):


     def __init__(				self     :						int     ,    UpperCAmelCase     :						str   )				->      Any:
      super().__init__(UpperCAmelCase   )
      __lowerCAmelCase:  List[Any]        = None
      __lowerCAmelCase:  Any        = []


     def     UpperCAmelCase    (				self     :						Dict   )				->      List[Any]:
      if self._dtype is None:
       raise ValueError('trainer_config_process() wasn\'t called yet to tell dtype'   )
      return self._dtype


     def     UpperCAmelCase    (				self     :						Union[str, Any]     ,    UpperCAmelCase     :						str   )				->      List[Any]:
      __lowerCAmelCase:  str        = self.get_value(UpperCAmelCase   )
      if val is None:
       return False
      else:
       return val == "auto"


     def     UpperCAmelCase    (				self     :						List[Any]     ,    UpperCAmelCase     :						Optional[int]     ,    UpperCAmelCase     :						List[Any]     ,    UpperCAmelCase     :						Union[str, Any]=None     ,    UpperCAmelCase     :						Optional[Any]=True   )				->      Dict:
      __lowerCAmelCase      ,				__lowerCAmelCase:  List[Any]        = self.find_config_node(UpperCAmelCase   )
      if config is None:
       return

      if config.get(UpperCAmelCase   ) == "auto":
       __lowerCAmelCase:  Optional[Any]        = hf_val
       return

      if not must_match:
       return

      __lowerCAmelCase:  Optional[Any]        = config.get(UpperCAmelCase   )
      if ds_val is not None and ds_val != hf_val:
       self.mismatches.append(F'''- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}'''   )

     _lowercase     :	List[str]        =      partialmethod(snake_case__				,      must_match=snake_case__   )


     def     UpperCAmelCase    (				self     :						int     ,    UpperCAmelCase     :						int   )				->      Any:
      __lowerCAmelCase:  List[Any]        = args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
      self.fill_match(
          'train_micro_batch_size_per_gpu'     ,    args.per_device_train_batch_size     ,    'per_device_train_batch_size'   )
      self.fill_match('gradient_accumulation_steps'     ,    args.gradient_accumulation_steps     ,    'gradient_accumulation_steps'   )
      self.fill_match('train_batch_size'     ,    UpperCAmelCase     ,    'train_batch_size (calculated)'   )
      self.fill_match('gradient_clipping'     ,    args.max_grad_norm     ,    'max_grad_norm'   )

      self.fill_match('optimizer.params.lr'     ,    args.learning_rate     ,    'learning_rate'   )
      self.fill_match('optimizer.params.betas'     ,    [args.adam_betaa, args.adam_betaa]     ,    'adam_beta1+adam_beta2'   )
      self.fill_match('optimizer.params.eps'     ,    args.adam_epsilon     ,    'adam_epsilon'   )
      self.fill_match('optimizer.params.weight_decay'     ,    args.weight_decay     ,    'weight_decay'   )

      self.fill_only('scheduler.params.warmup_min_lr'     ,    0   )  # not a trainer arg
      self.fill_match('scheduler.params.warmup_max_lr'     ,    args.learning_rate     ,    'learning_rate'   )
      # total_num_steps - will get set in trainer_config_finalize

      # fp16
      if args.fpaa or args.fpaa_full_eval:
       __lowerCAmelCase:  Any        = 'apex' if args.fpaa_backend == 'apex' else 'amp'
      else:
       __lowerCAmelCase:  Optional[Any]        = None

      if args.save_on_each_node:
       # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
       __lowerCAmelCase:  Optional[Any]        = self.config.get('checkpoint'     ,    {}   )
       __lowerCAmelCase:  List[str]        = args.save_on_each_node

      # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
      # any here unless the user did the work
      self.fill_match(
          'fp16.enabled'     ,    ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == 'amp')     ,    'fp16|fp16_full_eval+fp16_backend(amp)'     ,    )

      # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
      # ZeRO features
      self.fill_match('amp.enabled'     ,    fpaa_backend == 'apex'     ,    'fp16+fp16_backend(apex)'   )
      self.fill_match('amp.opt_level'     ,    args.fpaa_opt_level     ,    'fp16_opt_level'   )

      self.fill_match('bf16.enabled'     ,    (args.bfaa or args.bfaa_full_eval)     ,    'bf16|bf16_full_eval'   )

      # deepspeed's default mode is fp16 unless there is a config that says differently
      if self.is_true('bf16.enabled'   ):
       __lowerCAmelCase:  Union[str, Any]        = torch.bfloataa
      elif self.is_false('fp16.enabled'   ):
       __lowerCAmelCase:  List[str]        = torch.floataa
      else:
       __lowerCAmelCase:  List[str]        = torch.floataa


     def     UpperCAmelCase    (				self     :						List[Any]     ,    UpperCAmelCase     :						Optional[Any]     ,    UpperCAmelCase     :						Tuple     ,    UpperCAmelCase     :						Dict   )				->      Dict:
      __lowerCAmelCase:  List[str]        = [
          'zero_optimization.reduce_bucket_size',
          'zero_optimization.stage3_prefetch_bucket_size',
          'zero_optimization.stage3_param_persistence_threshold',
      ]
      __lowerCAmelCase:  Tuple        = [x for x in hidden_size_based_keys if self.is_auto(UpperCAmelCase   )]

      if len(UpperCAmelCase   ) > 0:
       if hasattr(model.config     ,    'hidden_size'   ):
        __lowerCAmelCase:  str        = model.config.hidden_size
       elif hasattr(model.config     ,    'hidden_sizes'   ):
        # if there are many hidden sizes pick the largest one
        __lowerCAmelCase:  List[str]        = max(model.config.hidden_sizes   )
       else:
        raise ValueError(
            'The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '
            'therefore it\'s not possible to automatically fill out the following `auto` entries '
            F'''in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '''
            '`auto` values for these keys with an integer value of your choice.'   )

       self.fill_only('zero_optimization.reduce_bucket_size'     ,    hidden_size * hidden_size   )
       if self.is_zeroa():
        # automatically assign the optimal config values based on model config
        self.fill_only('zero_optimization.stage3_prefetch_bucket_size'     ,    0.9 * hidden_size * hidden_size   )
        self.fill_only('zero_optimization.stage3_param_persistence_threshold'     ,    1_0 * hidden_size   )

        # scheduler
      self.fill_match('scheduler.params.total_num_steps'     ,    UpperCAmelCase     ,    'num_training_steps (calculated)'   )
      self.fill_match('scheduler.params.warmup_num_steps'     ,    args.get_warmup_steps(UpperCAmelCase   )     ,    'warmup_steps'   )

      if len(self.mismatches   ) > 0:
       __lowerCAmelCase:  Any        = '\n'.join(self.mismatches   )
       raise ValueError(
           'Please correct the following DeepSpeed config values that mismatch TrainingArguments'
           F''' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.'''   )







_a									=  None







def _a					(			SCREAMING_SNAKE_CASE						:     List[str]						)       ->       Optional[Any]:

 """simple docstring"""






 global _hf_deepspeed_config_weak_ref
 # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
 __lowerCAmelCase:  int        = weakref.ref(SCREAMING_SNAKE_CASE						)







def _a					(			)       ->       str:

 """simple docstring"""






 global _hf_deepspeed_config_weak_ref
 __lowerCAmelCase:  str        = None







def _a					(			)       ->       Optional[Any]:

 """simple docstring"""






 if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
  return _hf_deepspeed_config_weak_ref().is_zeroa()
 else:
  return False







def _a					(			)       ->       List[Any]:

 """simple docstring"""






 if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
  return _hf_deepspeed_config_weak_ref().config
 else:
  return None







def _a					(			SCREAMING_SNAKE_CASE						:     List[str]       ,   SCREAMING_SNAKE_CASE						:     Dict       ,   SCREAMING_SNAKE_CASE						:     Union[str, Any]       ,   SCREAMING_SNAKE_CASE						:     str       ,   SCREAMING_SNAKE_CASE						:     int						)       ->       List[str]:

 """simple docstring"""






 from accelerate.utils import DummyOptim, DummyScheduler

 __lowerCAmelCase:  Union[str, Any]        = hf_deepspeed_config.config

 # Optimizer + Scheduler
 # Currently supported combos:
 # 1. DS scheduler + DS optimizer: Yes
 # 2. HF scheduler + HF optimizer: Yes
 # 3. DS scheduler + HF optimizer: Yes
 # 4. HF scheduler + DS optimizer: No
 #
 # Unless Offload is enabled in which case it's:
 # 1. DS scheduler + DS optimizer: Yes
 # 2. HF scheduler + HF optimizer: Mostly*
 # 3. DS scheduler + HF optimizer: Mostly*
 # 4. HF scheduler + DS optimizer: No
 #
 # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

 __lowerCAmelCase:  Optional[int]        = None
 if "optimizer" in config:
  if args.adafactor:
   raise ValueError(
       '--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '
       'Only one optimizer can be configured.'						)
  __lowerCAmelCase:  List[Any]        = DummyOptim(params=SCREAMING_SNAKE_CASE						)
 else:
  if hf_deepspeed_config.is_offload():
   logger.info(
       'Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'
       ' custom optimizer has both CPU and GPU implementation (except LAMB)'						)

  # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
  # But trainer uses AdamW by default.
  __lowerCAmelCase:  Tuple        = trainer.create_optimizer()
  # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
  __lowerCAmelCase:  Union[str, Any]        = True

 __lowerCAmelCase:  int        = None
 if "scheduler" in config:
  __lowerCAmelCase:  Union[str, Any]        = DummyScheduler(SCREAMING_SNAKE_CASE						)
 else:
  if isinstance(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE						):
   raise ValueError(
       'Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '
       'Please configure a scheduler in the DeepSpeed config.'						)
  __lowerCAmelCase:  Union[str, Any]        = trainer.create_scheduler(num_training_steps=SCREAMING_SNAKE_CASE       ,   optimizer=SCREAMING_SNAKE_CASE						)

 return optimizer, lr_scheduler







def _a					(			SCREAMING_SNAKE_CASE						:     List[str]       ,   SCREAMING_SNAKE_CASE						:     Any       ,   SCREAMING_SNAKE_CASE						:     Tuple=False						)       ->       str:

 """simple docstring"""






 from deepspeed.utils import logger as ds_logger

 __lowerCAmelCase:  Tuple        = trainer.model
 __lowerCAmelCase:  Optional[int]        = trainer.args

 __lowerCAmelCase:  Any        = trainer.accelerator.state.deepspeed_plugin.hf_ds_config

 # resume config update - some bits like `model` and `num_training_steps` only become available during train
 hf_deepspeed_config.trainer_config_finalize(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE						)

 # set the Deepspeed log level consistent with the Trainer
 ds_logger.setLevel(args.get_process_log_level()						)

 if inference:
  # only Z3 makes sense for the inference
  if not hf_deepspeed_config.is_zeroa():
   raise ValueError('ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'						)

  # in case the training config is re-used for inference
  hf_deepspeed_config.del_config_sub_tree('optimizer'						)
  hf_deepspeed_config.del_config_sub_tree('lr_scheduler'						)
  __lowerCAmelCase      ,				__lowerCAmelCase:  str        = None, None
  __lowerCAmelCase:  Optional[int]        = None
 else:
  __lowerCAmelCase:  Union[str, Any]        = None  # important for when deepspeed_init is used as re-init
  __lowerCAmelCase:  List[Any]        = list(filter(lambda SCREAMING_SNAKE_CASE						: p.requires_grad       ,   model.parameters()						)						)
  __lowerCAmelCase      ,				__lowerCAmelCase:  str        = deepspeed_optim_sched(
      SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE						)

 # keep for quick debug:
 # from pprint import pprint; pprint(config)

 return optimizer, lr_scheduler







def _a					(			SCREAMING_SNAKE_CASE						:     Optional[int]       ,   SCREAMING_SNAKE_CASE						:     List[Any]						)       ->       Union[str, Any]:

 """simple docstring"""






 import glob

 __lowerCAmelCase:  Tuple        = sorted(glob.glob(f'''{checkpoint_path}/global_step*'''						)						)

 if len(SCREAMING_SNAKE_CASE						) > 0:
  logger.info(f'''Attempting to resume from {checkpoint_path}'''						)
  # this magically updates self.optimizer and self.lr_scheduler
  __lowerCAmelCase      ,				__lowerCAmelCase:  List[Any]        = deepspeed_engine.load_checkpoint(
      SCREAMING_SNAKE_CASE       ,   load_optimizer_states=SCREAMING_SNAKE_CASE       ,   load_lr_scheduler_states=SCREAMING_SNAKE_CASE						)
  if load_path is None:
   raise ValueError(f'''[deepspeed] failed to resume from checkpoint {checkpoint_path}'''						)
 else:
  raise ValueError(f'''Can\'t find a valid checkpoint at {checkpoint_path}'''						)








from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_a									=  logging.get_logger(__name__)

_a									=  {
    '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json''',
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}






class 				A_					(						snake_case__   ):
     _lowercase     :	List[Any]        =      'm2m_100'
     _lowercase     :	str        =      ['past_key_values']
     _lowercase     :	List[Any]        =      {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}


     def __init__(				self     :						int     ,    UpperCAmelCase     :						Any=1_2_8_1_1_2     ,    UpperCAmelCase     :						Optional[int]=1_0_2_4     ,    UpperCAmelCase     :						Tuple=1_2     ,    UpperCAmelCase     :						List[str]=4_0_9_6     ,    UpperCAmelCase     :						Dict=1_6     ,    UpperCAmelCase     :						int=1_2     ,    UpperCAmelCase     :						int=4_0_9_6     ,    UpperCAmelCase     :						List[str]=1_6     ,    UpperCAmelCase     :						Optional[Any]=0.05     ,    UpperCAmelCase     :						Dict=0.05     ,    UpperCAmelCase     :						Union[str, Any]=True     ,    UpperCAmelCase     :						Dict=True     ,    UpperCAmelCase     :						List[str]="relu"     ,    UpperCAmelCase     :						Dict=1_0_2_4     ,    UpperCAmelCase     :						str=0.1     ,    UpperCAmelCase     :						Dict=0.1     ,    UpperCAmelCase     :						List[str]=0.0     ,    UpperCAmelCase     :						Union[str, Any]=0.02     ,    UpperCAmelCase     :						Optional[Any]=2     ,    UpperCAmelCase     :						str=True     ,    UpperCAmelCase     :						Optional[int]=1     ,    UpperCAmelCase     :						List[Any]=0     ,    UpperCAmelCase     :						Union[str, Any]=2     ,    **UpperCAmelCase     :						Union[str, Any]     ,    )				->      Tuple:
      __lowerCAmelCase:  Optional[Any]        = vocab_size
      __lowerCAmelCase:  Dict        = max_position_embeddings
      __lowerCAmelCase:  Tuple        = d_model
      __lowerCAmelCase:  List[str]        = encoder_ffn_dim
      __lowerCAmelCase:  Any        = encoder_layers
      __lowerCAmelCase:  int        = encoder_attention_heads
      __lowerCAmelCase:  str        = decoder_ffn_dim
      __lowerCAmelCase:  Tuple        = decoder_layers
      __lowerCAmelCase:  Tuple        = decoder_attention_heads
      __lowerCAmelCase:  Any        = dropout
      __lowerCAmelCase:  Optional[int]        = attention_dropout
      __lowerCAmelCase:  Tuple        = activation_dropout
      __lowerCAmelCase:  Optional[Any]        = activation_function
      __lowerCAmelCase:  Union[str, Any]        = init_std
      __lowerCAmelCase:  List[str]        = encoder_layerdrop
      __lowerCAmelCase:  Optional[Any]        = decoder_layerdrop
      __lowerCAmelCase:  Dict        = use_cache
      __lowerCAmelCase:  Any        = encoder_layers
      __lowerCAmelCase:  Tuple        = scale_embedding  # scale factor will be sqrt(d_model) if True

      super().__init__(
          pad_token_id=UpperCAmelCase     ,    bos_token_id=UpperCAmelCase     ,    eos_token_id=UpperCAmelCase     ,    is_encoder_decoder=UpperCAmelCase     ,    decoder_start_token_id=UpperCAmelCase     ,    **UpperCAmelCase     ,    )







class 				A_					(						snake_case__   ):


     @property
     def     UpperCAmelCase    (				self     :						List[Any]   )				->      Mapping[str, Mapping[int, str]]:
      __lowerCAmelCase:  Union[str, Any]        = OrderedDict(
          [
              ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
              ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
          ]   )

      if self.use_past:
       __lowerCAmelCase:  str        = {0: 'batch'}
       __lowerCAmelCase:  Union[str, Any]        = {0: 'batch', 1: 'past_decoder_sequence + sequence'}
      else:
       __lowerCAmelCase:  Any        = {0: 'batch', 1: 'decoder_sequence'}
       __lowerCAmelCase:  Dict        = {0: 'batch', 1: 'decoder_sequence'}

      if self.use_past:
       self.fill_with_past_key_values_(UpperCAmelCase     ,    direction='inputs'   )
      return common_inputs


     def     UpperCAmelCase    (				self     :						Tuple     ,    UpperCAmelCase     :						PreTrainedTokenizer     ,    UpperCAmelCase     :						int = -1     ,    UpperCAmelCase     :						int = -1     ,    UpperCAmelCase     :						bool = False     ,    UpperCAmelCase     :						Optional[TensorType] = None     ,    )				->      Mapping[str, Any]:
      # Copied from OnnxConfig.generate_dummy_inputs
      # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
      # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
      __lowerCAmelCase:  List[str]        = compute_effective_axis_dimension(
          UpperCAmelCase     ,    fixed_dimension=OnnxConfig.default_fixed_batch     ,    num_token_to_add=0   )

      # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
      __lowerCAmelCase:  Tuple        = tokenizer.num_special_tokens_to_add(UpperCAmelCase   )
      __lowerCAmelCase:  List[str]        = compute_effective_axis_dimension(
          UpperCAmelCase     ,    fixed_dimension=OnnxConfig.default_fixed_sequence     ,    num_token_to_add=UpperCAmelCase   )

      # Generate dummy inputs according to compute batch and sequence
      __lowerCAmelCase:  int        = [' '.join([tokenizer.unk_token]   ) * seq_length] * batch_size
      __lowerCAmelCase:  Optional[int]        = dict(tokenizer(UpperCAmelCase     ,    return_tensors=UpperCAmelCase   )   )
      return common_inputs


     def     UpperCAmelCase    (				self     :						str     ,    UpperCAmelCase     :						PreTrainedTokenizer     ,    UpperCAmelCase     :						int = -1     ,    UpperCAmelCase     :						int = -1     ,    UpperCAmelCase     :						bool = False     ,    UpperCAmelCase     :						Optional[TensorType] = None     ,    )				->      Mapping[str, Any]:
      __lowerCAmelCase:  Optional[int]        = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
          UpperCAmelCase     ,    UpperCAmelCase     ,    UpperCAmelCase     ,    UpperCAmelCase     ,    UpperCAmelCase   )

      # Generate decoder inputs
      __lowerCAmelCase:  Any        = seq_length if not self.use_past else 1
      __lowerCAmelCase:  Union[str, Any]        = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
          UpperCAmelCase     ,    UpperCAmelCase     ,    UpperCAmelCase     ,    UpperCAmelCase     ,    UpperCAmelCase   )
      __lowerCAmelCase:  List[Any]        = {F'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
      __lowerCAmelCase:  List[str]        = dict(**UpperCAmelCase     ,    **UpperCAmelCase   )

      if self.use_past:
       if not is_torch_available():
        raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'   )
       else:
        import torch
       __lowerCAmelCase      ,				__lowerCAmelCase:  List[Any]        = common_inputs['input_ids'].shape
       __lowerCAmelCase:  int        = common_inputs['decoder_input_ids'].shape[1]
       __lowerCAmelCase      ,				__lowerCAmelCase:  str        = self.num_attention_heads
       __lowerCAmelCase:  str        = (
           batch,
           num_encoder_attention_heads,
           encoder_seq_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )
       __lowerCAmelCase:  Optional[int]        = decoder_seq_length + 3
       __lowerCAmelCase:  List[str]        = (
           batch,
           num_decoder_attention_heads,
           decoder_past_length,
           self._config.hidden_size // num_decoder_attention_heads,
       )

       __lowerCAmelCase:  Dict        = torch.cat(
           [common_inputs['decoder_attention_mask'], torch.ones(UpperCAmelCase     ,    UpperCAmelCase   )]     ,    dim=1   )

       __lowerCAmelCase:  str        = []
       # If the number of encoder and decoder layers are present in the model configuration, both are considered
       __lowerCAmelCase      ,				__lowerCAmelCase:  str        = self.num_layers
       __lowerCAmelCase:  Tuple        = min(UpperCAmelCase     ,    UpperCAmelCase   )
       __lowerCAmelCase:  List[Any]        = max(UpperCAmelCase     ,    UpperCAmelCase   ) - min_num_layers
       __lowerCAmelCase:  Any        = 'encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

       for _ in range(UpperCAmelCase   ):
        common_inputs["past_key_values"].append(
            (
                torch.zeros(UpperCAmelCase   ),
                torch.zeros(UpperCAmelCase   ),
                torch.zeros(UpperCAmelCase   ),
                torch.zeros(UpperCAmelCase   ),
            )   )
       # TODO: test this.
       __lowerCAmelCase:  List[Any]        = encoder_shape if remaining_side_name == 'encoder' else decoder_shape
       for _ in range(UpperCAmelCase     ,    UpperCAmelCase   ):
        common_inputs["past_key_values"].append((torch.zeros(UpperCAmelCase   ), torch.zeros(UpperCAmelCase   ))   )
      return common_inputs


     _lowercase     :	int        =      _generate_dummy_inputs_for_default_and_seqaseq_lm





