
import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     MODEL_FOR_PRETRAINING_MAPPING,
     FunnelBaseModel,
     FunnelForMaskedLM,
     FunnelForMultipleChoice,
     FunnelForPreTraining,
     FunnelForQuestionAnswering,
     FunnelForSequenceClassification,
     FunnelForTokenClassification,
     FunnelModel,
 )


class 			UpperCamelCase_				:




      '''simple docstring'''







      def __init__( self					,					_UpperCAmelCase					,					_UpperCAmelCase=13					,					_UpperCAmelCase=7					,					_UpperCAmelCase=True					,					_UpperCAmelCase=True					,					_UpperCAmelCase=True					,					_UpperCAmelCase=True					,					_UpperCAmelCase=99					,					_UpperCAmelCase=[1, 1, 2]					,					_UpperCAmelCase=1					,					_UpperCAmelCase=32					,					_UpperCAmelCase=4					,					_UpperCAmelCase=8					,					_UpperCAmelCase=37					,					_UpperCAmelCase="gelu_new"					,					_UpperCAmelCase=0.1					,					_UpperCAmelCase=0.1					,					_UpperCAmelCase=0.0					,					_UpperCAmelCase=512					,					_UpperCAmelCase=3					,					_UpperCAmelCase=0.02					,					_UpperCAmelCase=3					,					_UpperCAmelCase=4					,					_UpperCAmelCase=None					,					_UpperCAmelCase=False					,					):
            lowerCAmelCase_						=      parent
            lowerCAmelCase_						=      batch_size
            lowerCAmelCase_						=      seq_length
            lowerCAmelCase_						=      is_training
            lowerCAmelCase_						=      use_input_mask
            lowerCAmelCase_						=      use_token_type_ids
            lowerCAmelCase_						=      use_labels
            lowerCAmelCase_						=      vocab_size
            lowerCAmelCase_						=      block_sizes
            lowerCAmelCase_						=      num_decoder_layers
            lowerCAmelCase_						=      d_model
            lowerCAmelCase_						=      n_head
            lowerCAmelCase_						=      d_head
            lowerCAmelCase_						=      d_inner
            lowerCAmelCase_						=      hidden_act
            lowerCAmelCase_						=      hidden_dropout
            lowerCAmelCase_						=      attention_dropout
            lowerCAmelCase_						=      activation_dropout
            lowerCAmelCase_						=      max_position_embeddings
            lowerCAmelCase_						=      type_vocab_size
            lowerCAmelCase_						=      2
            lowerCAmelCase_						=      num_labels
            lowerCAmelCase_						=      num_choices
            lowerCAmelCase_						=      scope
            lowerCAmelCase_						=      initializer_std

            # Used in the tests to check the size of the first attention layer
            lowerCAmelCase_						=      n_head
            # Used in the tests to check the size of the first hidden state
            lowerCAmelCase_						=      self.d_model
            # Used in the tests to check the number of output hidden states/attentions
            lowerCAmelCase_						=      sum(self.block_sizes) + (0 if base else self.num_decoder_layers)
            # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
            # the last hidden state of the first block (which is the first hidden state of the decoder).
            if not base:
                  lowerCAmelCase_						=      self.num_hidden_layers + 2







      def lowercase__		( self):
            lowerCAmelCase_						=      ids_tensor([self.batch_size, self.seq_length]					,					self.vocab_size)

            lowerCAmelCase_						=      None
            if self.use_input_mask:
                  lowerCAmelCase_						=      ids_tensor([self.batch_size, self.seq_length]					,					vocab_size=2)

            lowerCAmelCase_						=      None
            if self.use_token_type_ids:
                  lowerCAmelCase_						=      ids_tensor([self.batch_size, self.seq_length]					,					self.type_vocab_size)

            lowerCAmelCase_						=      None
            lowerCAmelCase_						=      None
            lowerCAmelCase_						=      None
            if self.use_labels:
                  lowerCAmelCase_						=      ids_tensor([self.batch_size]					,					self.type_sequence_label_size)
                  lowerCAmelCase_						=      ids_tensor([self.batch_size, self.seq_length]					,					self.num_labels)
                  lowerCAmelCase_						=      ids_tensor([self.batch_size]					,					self.num_choices)
                  lowerCAmelCase_						=      ids_tensor([self.batch_size, self.seq_length]					,					1)

            lowerCAmelCase_						=      self.get_config()

            return (
                config,
                input_ids,
                token_type_ids,
                input_mask,
                sequence_labels,
                token_labels,
                choice_labels,
                fake_token_labels,
            )







      def lowercase__		( self):
            return FunnelConfig(
                vocab_size=self.vocab_size					,					block_sizes=self.block_sizes					,					num_decoder_layers=self.num_decoder_layers					,					d_model=self.d_model					,					n_head=self.n_head					,					d_head=self.d_head					,					d_inner=self.d_inner					,					hidden_act=self.hidden_act					,					hidden_dropout=self.hidden_dropout					,					attention_dropout=self.attention_dropout					,					activation_dropout=self.activation_dropout					,					max_position_embeddings=self.max_position_embeddings					,					type_vocab_size=self.type_vocab_size					,					initializer_std=self.initializer_std					,					)







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					):
            lowerCAmelCase_						=      FunnelModel(config=_UpperCAmelCase)
            model.to(_UpperCAmelCase)
            model.eval()
            lowerCAmelCase_						=      model(_UpperCAmelCase					,					attention_mask=_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase)
            lowerCAmelCase_						=      model(_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase)
            lowerCAmelCase_						=      model(_UpperCAmelCase)
            self.parent.assertEqual(result.last_hidden_state.shape					,					(self.batch_size, self.seq_length, self.d_model))

            lowerCAmelCase_						=      False
            lowerCAmelCase_						=      model(_UpperCAmelCase)
            self.parent.assertEqual(result.last_hidden_state.shape					,					(self.batch_size, self.seq_length, self.d_model))

            lowerCAmelCase_						=      False
            lowerCAmelCase_						=      model(_UpperCAmelCase)
            self.parent.assertEqual(result.last_hidden_state.shape					,					(self.batch_size, self.seq_length, self.d_model))







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					):
            lowerCAmelCase_						=      FunnelBaseModel(config=_UpperCAmelCase)
            model.to(_UpperCAmelCase)
            model.eval()
            lowerCAmelCase_						=      model(_UpperCAmelCase					,					attention_mask=_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase)
            lowerCAmelCase_						=      model(_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase)
            lowerCAmelCase_						=      model(_UpperCAmelCase)
            self.parent.assertEqual(result.last_hidden_state.shape					,					(self.batch_size, 2, self.d_model))

            lowerCAmelCase_						=      False
            lowerCAmelCase_						=      model(_UpperCAmelCase)
            self.parent.assertEqual(result.last_hidden_state.shape					,					(self.batch_size, 3, self.d_model))

            lowerCAmelCase_						=      False
            lowerCAmelCase_						=      model(_UpperCAmelCase)
            self.parent.assertEqual(result.last_hidden_state.shape					,					(self.batch_size, 2, self.d_model))







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					):
            lowerCAmelCase_						=      self.num_labels
            lowerCAmelCase_						=      FunnelForPreTraining(config=_UpperCAmelCase)
            model.to(_UpperCAmelCase)
            model.eval()
            lowerCAmelCase_						=      model(_UpperCAmelCase					,					attention_mask=_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase					,					labels=_UpperCAmelCase)
            self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.seq_length))







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					):
            lowerCAmelCase_						=      FunnelForMaskedLM(config=_UpperCAmelCase)
            model.to(_UpperCAmelCase)
            model.eval()
            lowerCAmelCase_						=      model(_UpperCAmelCase					,					attention_mask=_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase					,					labels=_UpperCAmelCase)
            self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.seq_length, self.vocab_size))







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					):
            lowerCAmelCase_						=      self.num_labels
            lowerCAmelCase_						=      FunnelForSequenceClassification(_UpperCAmelCase)
            model.to(_UpperCAmelCase)
            model.eval()
            lowerCAmelCase_						=      model(_UpperCAmelCase					,					attention_mask=_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase					,					labels=_UpperCAmelCase)
            self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.num_labels))







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					):
            lowerCAmelCase_						=      self.num_choices
            lowerCAmelCase_						=      FunnelForMultipleChoice(config=_UpperCAmelCase)
            model.to(_UpperCAmelCase)
            model.eval()
            lowerCAmelCase_						=      input_ids.unsqueeze(1).expand(-1					,					self.num_choices					,					-1).contiguous()
            lowerCAmelCase_						=      token_type_ids.unsqueeze(1).expand(-1					,					self.num_choices					,					-1).contiguous()
            lowerCAmelCase_						=      input_mask.unsqueeze(1).expand(-1					,					self.num_choices					,					-1).contiguous()
            lowerCAmelCase_						=      model(
                _UpperCAmelCase					,					attention_mask=_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase					,					labels=_UpperCAmelCase					,					)
            self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.num_choices))







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					):
            lowerCAmelCase_						=      self.num_labels
            lowerCAmelCase_						=      FunnelForTokenClassification(config=_UpperCAmelCase)
            model.to(_UpperCAmelCase)
            model.eval()
            lowerCAmelCase_						=      model(_UpperCAmelCase					,					attention_mask=_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase					,					labels=_UpperCAmelCase)
            self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.seq_length, self.num_labels))







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					):
            lowerCAmelCase_						=      FunnelForQuestionAnswering(config=_UpperCAmelCase)
            model.to(_UpperCAmelCase)
            model.eval()
            lowerCAmelCase_						=      model(
                _UpperCAmelCase					,					attention_mask=_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase					,					start_positions=_UpperCAmelCase					,					end_positions=_UpperCAmelCase					,					)
            self.parent.assertEqual(result.start_logits.shape					,					(self.batch_size, self.seq_length))
            self.parent.assertEqual(result.end_logits.shape					,					(self.batch_size, self.seq_length))







      def lowercase__		( self):
            lowerCAmelCase_						=      self.prepare_config_and_inputs()
            (
                (
                lowerCAmelCase_
            )			,		(
                lowerCAmelCase_
            )			,		(
                lowerCAmelCase_
            )			,		(
                lowerCAmelCase_
            )			,		(
                lowerCAmelCase_
            )			,		(
                lowerCAmelCase_
            )			,		(
                lowerCAmelCase_
            )			,		(
                lowerCAmelCase_
            )			,		
            )						=      config_and_inputs
            lowerCAmelCase_						=      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
            return config, inputs_dict


@require_torch
class 			UpperCamelCase_				(  A					,       A					,       unittest.TestCase     ):




      '''simple docstring'''







      a		:Any						 =      False
      a		:int						 =      False
      a		:List[str]						 =      (
          (
              FunnelModel,
              FunnelForMaskedLM,
              FunnelForPreTraining,
              FunnelForQuestionAnswering,
              FunnelForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      a		:Union[str, Any]						 =      (
          {
              'feature-extraction': (FunnelBaseModel, FunnelModel),
              'fill-mask': FunnelForMaskedLM,
              'question-answering': FunnelForQuestionAnswering,
              'text-classification': FunnelForSequenceClassification,
              'token-classification': FunnelForTokenClassification,
              'zero-shot': FunnelForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase=False):
            lowerCAmelCase_						=      super()._prepare_for_class(_UpperCAmelCase					,					_UpperCAmelCase					,					return_labels=_UpperCAmelCase)

            if return_labels:
                  if model_class in get_values(_UpperCAmelCase):
                        lowerCAmelCase_						=      torch.zeros(
                            (self.model_tester.batch_size, self.model_tester.seq_length)					,					dtype=torch.long					,					device=_UpperCAmelCase)
            return inputs_dict







      def lowercase__		( self):
            lowerCAmelCase_						=      FunnelModelTester(self)
            lowerCAmelCase_						=      ConfigTester(self					,					config_class=_UpperCAmelCase)







      def lowercase__		( self):
            self.config_tester.run_common_tests()







      def lowercase__		( self):
            lowerCAmelCase_						=      self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_model(*_UpperCAmelCase)







      def lowercase__		( self):
            lowerCAmelCase_						=      self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_pretraining(*_UpperCAmelCase)







      def lowercase__		( self):
            lowerCAmelCase_						=      self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase)







      def lowercase__		( self):
            lowerCAmelCase_						=      self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase)







      def lowercase__		( self):
            lowerCAmelCase_						=      self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase)







      def lowercase__		( self					,					_UpperCAmelCase):
            if hasattr(_UpperCAmelCase					,					'''weight''') and module.weight is not None:
                  module.weight.data.fill_(3)
            if hasattr(_UpperCAmelCase					,					'''bias''') and module.bias is not None:
                  module.bias.data.fill_(3)

            for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
                  if hasattr(_UpperCAmelCase					,					_UpperCAmelCase) and getattr(_UpperCAmelCase					,					_UpperCAmelCase) is not None:
                        lowerCAmelCase_						=      getattr(_UpperCAmelCase					,					_UpperCAmelCase)
                        weight.data.fill_(3)


@require_torch
class 			UpperCamelCase_				(  A					,       unittest.TestCase     ):




      '''simple docstring'''







      a		:Optional[int]						 =      False
      a		:Tuple						 =      False
      a		:Any						 =      (
          (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
      )







      def lowercase__		( self):
            lowerCAmelCase_						=      FunnelModelTester(self					,					base=_UpperCAmelCase)
            lowerCAmelCase_						=      ConfigTester(self					,					config_class=_UpperCAmelCase)







      def lowercase__		( self):
            self.config_tester.run_common_tests()







      def lowercase__		( self):
            lowerCAmelCase_						=      self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_base_model(*_UpperCAmelCase)







      def lowercase__		( self):
            lowerCAmelCase_						=      self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase)







      def lowercase__		( self):
            lowerCAmelCase_						=      self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase)







      def lowercase__		( self):
            lowerCAmelCase_			,		lowerCAmelCase_						=      self.model_tester.prepare_config_and_inputs_for_common()
            lowerCAmelCase_						=      True

            for model_class in self.all_model_classes:
                  if model_class.__name__ == "FunnelBaseModel":
                        continue
                  lowerCAmelCase_						=      model_class(_UpperCAmelCase)
                  model.to(_UpperCAmelCase)
                  model.train()
                  lowerCAmelCase_						=      self._prepare_for_class(_UpperCAmelCase					,					_UpperCAmelCase					,					return_labels=_UpperCAmelCase)
                  lowerCAmelCase_						=      model(**_UpperCAmelCase).loss
                  loss.backward()







      def lowercase__		( self					,					_UpperCAmelCase):
            if hasattr(_UpperCAmelCase					,					'''weight''') and module.weight is not None:
                  module.weight.data.fill_(3)
            if hasattr(_UpperCAmelCase					,					'''bias''') and module.bias is not None:
                  module.bias.data.fill_(3)

            for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
                  if hasattr(_UpperCAmelCase					,					_UpperCAmelCase) and getattr(_UpperCAmelCase					,					_UpperCAmelCase) is not None:
                        lowerCAmelCase_						=      getattr(_UpperCAmelCase					,					_UpperCAmelCase)
                        weight.data.fill_(3)


@require_torch
@require_sentencepiece
@require_tokenizers
class 			UpperCamelCase_				(  unittest.TestCase     ):




      '''simple docstring'''







      def lowercase__		( self):
            lowerCAmelCase_						=      13
            lowerCAmelCase_						=      7
            lowerCAmelCase_						=      torch.arange(0					,					batch_size * sequence_length).long().reshape(_UpperCAmelCase					,					_UpperCAmelCase)
            lowerCAmelCase_						=      [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
            lowerCAmelCase_						=      torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths])

            lowerCAmelCase_						=      FunnelModel.from_pretrained('''sgugger/funnel-random-tiny''')
            lowerCAmelCase_						=      model(_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase)[0].abs()

            lowerCAmelCase_						=      torch.tensor(2344.8352)
            lowerCAmelCase_						=      torch.tensor(0.8052)
            self.assertTrue(torch.allclose(output.sum()					,					_UpperCAmelCase					,					atol=1E-4))
            self.assertTrue(torch.allclose(output.mean()					,					_UpperCAmelCase					,					atol=1E-4))

            lowerCAmelCase_						=      torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]])
            lowerCAmelCase_						=      model(_UpperCAmelCase					,					attention_mask=_UpperCAmelCase					,					token_type_ids=_UpperCAmelCase)[0].abs()

            lowerCAmelCase_						=      torch.tensor(2343.8425)
            lowerCAmelCase_						=      torch.tensor(0.8049)
            self.assertTrue(torch.allclose(output.sum()					,					_UpperCAmelCase					,					atol=1E-4))
            self.assertTrue(torch.allclose(output.mean()					,					_UpperCAmelCase					,					atol=1E-4))







      @slow
      def lowercase__		( self):
            lowerCAmelCase_						=      FunnelTokenizer.from_pretrained('''huggingface/funnel-small''')
            lowerCAmelCase_						=      FunnelModel.from_pretrained('''huggingface/funnel-small''')
            lowerCAmelCase_						=      tokenizer('''Hello! I am the Funnel Transformer model.'''					,					return_tensors='''pt''')
            lowerCAmelCase_						=      model(**_UpperCAmelCase)[0]

            lowerCAmelCase_						=      torch.tensor(235.7246)
            lowerCAmelCase_						=      torch.tensor(0.0256)
            self.assertTrue(torch.allclose(output.sum()					,					_UpperCAmelCase					,					atol=1E-4))
            self.assertTrue(torch.allclose(output.mean()					,					_UpperCAmelCase					,					atol=1E-4))





from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


_snake_case		  =   logging.get_logger(__name__)

_snake_case		  =   {
    "facebook/blenderbot-3B": "https://huggingface.co/facebook/blenderbot-3B/resolve/main/config.json",
    # See all Blenderbot models at https://huggingface.co/models?filter=blenderbot
}


class 			UpperCamelCase_				(  A     ):




      '''simple docstring'''







      a		:Union[str, Any]						 =      'blenderbot'
      a		:List[str]						 =      ['past_key_values']
      a		:Any						 =      {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}







      def __init__( self					,					_UpperCAmelCase=8_008					,					_UpperCAmelCase=128					,					_UpperCAmelCase=2					,					_UpperCAmelCase=10_240					,					_UpperCAmelCase=32					,					_UpperCAmelCase=24					,					_UpperCAmelCase=10_240					,					_UpperCAmelCase=32					,					_UpperCAmelCase=0.0					,					_UpperCAmelCase=0.0					,					_UpperCAmelCase=True					,					_UpperCAmelCase=True					,					_UpperCAmelCase="gelu"					,					_UpperCAmelCase=2_560					,					_UpperCAmelCase=0.1					,					_UpperCAmelCase=0.0					,					_UpperCAmelCase=0.0					,					_UpperCAmelCase=0.02					,					_UpperCAmelCase=1					,					_UpperCAmelCase=False					,					_UpperCAmelCase=0					,					_UpperCAmelCase=1					,					_UpperCAmelCase=2					,					_UpperCAmelCase=3					,					_UpperCAmelCase=2					,					**_UpperCAmelCase					,					):
            lowerCAmelCase_						=      vocab_size
            lowerCAmelCase_						=      max_position_embeddings
            lowerCAmelCase_						=      d_model
            lowerCAmelCase_						=      encoder_ffn_dim
            lowerCAmelCase_						=      encoder_layers
            lowerCAmelCase_						=      encoder_attention_heads
            lowerCAmelCase_						=      decoder_ffn_dim
            lowerCAmelCase_						=      decoder_layers
            lowerCAmelCase_						=      decoder_attention_heads
            lowerCAmelCase_						=      dropout
            lowerCAmelCase_						=      attention_dropout
            lowerCAmelCase_						=      activation_dropout
            lowerCAmelCase_						=      activation_function
            lowerCAmelCase_						=      init_std
            lowerCAmelCase_						=      encoder_layerdrop
            lowerCAmelCase_						=      decoder_layerdrop
            lowerCAmelCase_						=      use_cache
            lowerCAmelCase_						=      encoder_layers
            lowerCAmelCase_						=      scale_embedding  # scale factor will be sqrt(d_model) if True

            super().__init__(
                pad_token_id=_UpperCAmelCase					,					bos_token_id=_UpperCAmelCase					,					eos_token_id=_UpperCAmelCase					,					is_encoder_decoder=_UpperCAmelCase					,					decoder_start_token_id=_UpperCAmelCase					,					encoder_no_repeat_ngram_size=_UpperCAmelCase					,					forced_eos_token_id=_UpperCAmelCase					,					**_UpperCAmelCase					,					)


class 			UpperCamelCase_				(  A     ):




      '''simple docstring'''







      @property
      def lowercase__		( self):
            if self.task in ["default", "seq2seq-lm"]:
                  lowerCAmelCase_						=      OrderedDict(
                      [
                          ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                          ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                      ])
                  if self.use_past:
                        lowerCAmelCase_						=      {0: '''batch'''}
                        lowerCAmelCase_						=      {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
                  else:
                        lowerCAmelCase_						=      {0: '''batch''', 1: '''decoder_sequence'''}
                        lowerCAmelCase_						=      {0: '''batch''', 1: '''decoder_sequence'''}
                  if self.use_past:
                        self.fill_with_past_key_values_(_UpperCAmelCase					,					direction='''inputs''')
            elif self.task == "causal-lm":
                  lowerCAmelCase_						=      OrderedDict(
                      [
                          ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                          ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                      ])
                  if self.use_past:
                        lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_layers
                        for i in range(_UpperCAmelCase):
                              lowerCAmelCase_						=      {0: '''batch''', 2: '''past_sequence + sequence'''}
                              lowerCAmelCase_						=      {0: '''batch''', 2: '''past_sequence + sequence'''}
            else:
                  lowerCAmelCase_						=      OrderedDict(
                      [
                          ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                          ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                          ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
                          ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
                      ])

            return common_inputs







      @property
      # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
      def lowercase__		( self):
            if self.task in ["default", "seq2seq-lm"]:
                  lowerCAmelCase_						=      super().outputs
            else:
                  lowerCAmelCase_						=      super(_UpperCAmelCase					,					self).outputs
                  if self.use_past:
                        lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_layers
                        for i in range(_UpperCAmelCase):
                              lowerCAmelCase_						=      {0: '''batch''', 2: '''past_sequence + sequence'''}
                              lowerCAmelCase_						=      {0: '''batch''', 2: '''past_sequence + sequence'''}
            return common_outputs







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = False					,					_UpperCAmelCase = None					,					):
            lowerCAmelCase_						=      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                _UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase)
            # Generate decoder inputs
            lowerCAmelCase_						=      seq_length if not self.use_past else 1
            lowerCAmelCase_						=      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                _UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase)
            lowerCAmelCase_						=      {f'decoder_{name}': tensor for name, tensor in decoder_inputs.items()}
            lowerCAmelCase_						=      dict(**_UpperCAmelCase					,					**_UpperCAmelCase)

            if self.use_past:
                  if not is_torch_available():
                        raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.''')
                  else:
                        import torch
                  lowerCAmelCase_			,		lowerCAmelCase_						=      common_inputs['''input_ids'''].shape
                  lowerCAmelCase_						=      common_inputs['''decoder_input_ids'''].shape[1]
                  lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_attention_heads
                  lowerCAmelCase_						=      (
                      batch,
                      num_encoder_attention_heads,
                      encoder_seq_length,
                      self._config.hidden_size // num_encoder_attention_heads,
                  )
                  lowerCAmelCase_						=      decoder_seq_length
                  lowerCAmelCase_						=      (
                      batch,
                      num_decoder_attention_heads,
                      decoder_past_length,
                      self._config.hidden_size // num_decoder_attention_heads,
                  )
                  lowerCAmelCase_						=      torch.cat(
                      [common_inputs['''decoder_attention_mask'''], torch.ones(_UpperCAmelCase					,					_UpperCAmelCase)]					,					dim=1)
                  lowerCAmelCase_						=      []
                  lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_layers

                  for _ in range(_UpperCAmelCase):
                        common_inputs["past_key_values"].append(
                            (
                                torch.zeros(_UpperCAmelCase),
                                torch.zeros(_UpperCAmelCase),
                                torch.zeros(_UpperCAmelCase),
                                torch.zeros(_UpperCAmelCase),
                            ))
            return common_inputs







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = False					,					_UpperCAmelCase = None					,					):
            lowerCAmelCase_						=      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                _UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase)

            if self.use_past:
                  if not is_torch_available():
                        raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.''')
                  else:
                        import torch
                  lowerCAmelCase_			,		lowerCAmelCase_						=      common_inputs['''input_ids'''].shape
                  lowerCAmelCase_						=      seqlen
                  lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_layers
                  lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_attention_heads
                  lowerCAmelCase_						=      (
                      batch,
                      num_encoder_attention_heads,
                      past_key_values_length,
                      self._config.hidden_size // num_encoder_attention_heads,
                  )
                  lowerCAmelCase_						=      common_inputs['''attention_mask'''].dtype
                  lowerCAmelCase_						=      torch.cat(
                      [common_inputs['''attention_mask'''], torch.ones(_UpperCAmelCase					,					_UpperCAmelCase					,					dtype=_UpperCAmelCase)]					,					dim=1)
                  lowerCAmelCase_						=      [
                      (torch.zeros(_UpperCAmelCase), torch.zeros(_UpperCAmelCase)) for _ in range(_UpperCAmelCase)
                  ]
            return common_inputs







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = False					,					_UpperCAmelCase = None					,					):
            # Copied from OnnxConfig.generate_dummy_inputs
            # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
            # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
            lowerCAmelCase_						=      compute_effective_axis_dimension(
                _UpperCAmelCase					,					fixed_dimension=OnnxConfig.default_fixed_batch					,					num_token_to_add=0)

            # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
            lowerCAmelCase_						=      tokenizer.num_special_tokens_to_add(_UpperCAmelCase)
            lowerCAmelCase_						=      compute_effective_axis_dimension(
                _UpperCAmelCase					,					fixed_dimension=OnnxConfig.default_fixed_sequence					,					num_token_to_add=_UpperCAmelCase)

            # Generate dummy inputs according to compute batch and sequence
            lowerCAmelCase_						=      [''' '''.join([tokenizer.unk_token]) * seq_length] * batch_size
            lowerCAmelCase_						=      dict(tokenizer(_UpperCAmelCase					,					return_tensors=_UpperCAmelCase))
            return common_inputs







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = -1					,					_UpperCAmelCase = False					,					_UpperCAmelCase = None					,					):
            if self.task in ["default", "seq2seq-lm"]:
                  lowerCAmelCase_						=      self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                      _UpperCAmelCase					,					batch_size=_UpperCAmelCase					,					seq_length=_UpperCAmelCase					,					is_pair=_UpperCAmelCase					,					framework=_UpperCAmelCase)

            elif self.task == "causal-lm":
                  lowerCAmelCase_						=      self._generate_dummy_inputs_for_causal_lm(
                      _UpperCAmelCase					,					batch_size=_UpperCAmelCase					,					seq_length=_UpperCAmelCase					,					is_pair=_UpperCAmelCase					,					framework=_UpperCAmelCase)
            else:
                  lowerCAmelCase_						=      self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                      _UpperCAmelCase					,					batch_size=_UpperCAmelCase					,					seq_length=_UpperCAmelCase					,					is_pair=_UpperCAmelCase					,					framework=_UpperCAmelCase)

            return common_inputs







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase):
            if self.task in ["default", "seq2seq-lm"]:
                  lowerCAmelCase_						=      super()._flatten_past_key_values_(_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase)
            else:
                  lowerCAmelCase_						=      super(_UpperCAmelCase					,					self)._flatten_past_key_values_(
                      _UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase					,					_UpperCAmelCase)







      def lowercase__		( self					,					_UpperCAmelCase					,					_UpperCAmelCase):
            if direction not in ["inputs", "outputs"]:
                  raise ValueError(f'direction must either be "inputs" or "outputs", but {direction} was given')

            lowerCAmelCase_						=      '''past_key_values''' if direction == '''inputs''' else '''present'''
            lowerCAmelCase_			,		lowerCAmelCase_						=      self.num_layers

            lowerCAmelCase_						=      '''past_encoder_sequence'''
            lowerCAmelCase_						=      '''past_decoder_sequence''' if direction == '''inputs''' else '''past_decoder_sequence + sequence'''

            for i in range(_UpperCAmelCase):
                  lowerCAmelCase_						=      {0: '''batch''', 2: decoder_sequence}
                  lowerCAmelCase_						=      {0: '''batch''', 2: decoder_sequence}
                  lowerCAmelCase_						=      {0: '''batch''', 2: encoder_sequence}
                  lowerCAmelCase_						=      {0: '''batch''', 2: encoder_sequence}



