

"""simple docstring"""





import unittest

from transformers import ElectraConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    ElectraForCausalLM,
							    ElectraForMaskedLM,
							    ElectraForMultipleChoice,
							    ElectraForPreTraining,
							    ElectraForQuestionAnswering,
							    ElectraForSequenceClassification,
							    ElectraForTokenClassification,
							    ElectraModel,
							)
							from transformers.models.electra.modeling_electra import ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST







class        SCREAMING_SNAKE_CASE      :






					"""simple docstring"""



					def __init__( self    :       List[Any]     ,lowercase_    :       str     ,lowercase_    :       Any=1_3     ,lowercase_    :       Union[str, Any]=7     ,lowercase_    :       List[str]=True     ,lowercase_    :       Optional[int]=True     ,lowercase_    :       List[Any]=True     ,lowercase_    :       List[str]=True     ,lowercase_    :       Optional[int]=9_9     ,lowercase_    :       Union[str, Any]=3_2     ,lowercase_    :       Union[str, Any]=5     ,lowercase_    :       Optional[int]=4     ,lowercase_    :       Tuple=3_7     ,lowercase_    :       List[Any]="gelu"     ,lowercase_    :       Any=0.1     ,lowercase_    :       Any=0.1     ,lowercase_    :       List[str]=5_1_2     ,lowercase_    :       Tuple=1_6     ,lowercase_    :       int=2     ,lowercase_    :       str=0.02     ,lowercase_    :       Optional[Any]=3     ,lowercase_    :       Any=4     ,lowercase_    :       int=None     ,):
						lowerCAmelCase__			:		List[str]						       =			parent
						lowerCAmelCase__			:		str						       =			batch_size
						lowerCAmelCase__			:		Tuple						       =			seq_length
						lowerCAmelCase__			:		Optional[Any]						       =			is_training
						lowerCAmelCase__			:		List[str]						       =			use_input_mask
						lowerCAmelCase__			:		List[str]						       =			use_token_type_ids
						lowerCAmelCase__			:		int						       =			use_labels
						lowerCAmelCase__			:		List[Any]						       =			vocab_size
						lowerCAmelCase__			:		List[Any]						       =			hidden_size
						lowerCAmelCase__			:		Union[str, Any]						       =			num_hidden_layers
						lowerCAmelCase__			:		Tuple						       =			num_attention_heads
						lowerCAmelCase__			:		List[str]						       =			intermediate_size
						lowerCAmelCase__			:		List[Any]						       =			hidden_act
						lowerCAmelCase__			:		str						       =			hidden_dropout_prob
						lowerCAmelCase__			:		Any						       =			attention_probs_dropout_prob
						lowerCAmelCase__			:		Tuple						       =			max_position_embeddings
						lowerCAmelCase__			:		Tuple						       =			type_vocab_size
						lowerCAmelCase__			:		Union[str, Any]						       =			type_sequence_label_size
						lowerCAmelCase__			:		Union[str, Any]						       =			initializer_range
						lowerCAmelCase__			:		str						       =			num_labels
						lowerCAmelCase__			:		Dict						       =			num_choices
						lowerCAmelCase__			:		Any						       =			scope







					def  __lowerCAmelCase				( self    :       Optional[int]	):
						lowerCAmelCase__			:		int						       =			ids_tensor([self.batch_size, self.seq_length]     ,self.vocab_size	)

						lowerCAmelCase__			:		List[str]						       =			None
						if self.use_input_mask:
							lowerCAmelCase__			:		Dict						       =			random_attention_mask([self.batch_size, self.seq_length]	)

						lowerCAmelCase__			:		Union[str, Any]						       =			None
						if self.use_token_type_ids:
							lowerCAmelCase__			:		str						       =			ids_tensor([self.batch_size, self.seq_length]     ,self.type_vocab_size	)

						lowerCAmelCase__			:		List[str]						       =			None
						lowerCAmelCase__			:		str						       =			None
						lowerCAmelCase__			:		Tuple						       =			None
						if self.use_labels:
							lowerCAmelCase__			:		Optional[int]						       =			ids_tensor([self.batch_size]     ,self.type_sequence_label_size	)
							lowerCAmelCase__			:		str						       =			ids_tensor([self.batch_size, self.seq_length]     ,self.num_labels	)
							lowerCAmelCase__			:		Union[str, Any]						       =			ids_tensor([self.batch_size]     ,self.num_choices	)
							lowerCAmelCase__			:		Tuple						       =			ids_tensor([self.batch_size, self.seq_length]     ,1	)

						lowerCAmelCase__			:		List[Any]						       =			self.get_config()

						return (
						    config,
						    input_ids,
						    token_type_ids,
						    input_mask,
						    sequence_labels,
						    token_labels,
						    choice_labels,
						    fake_token_labels,
						)







					def  __lowerCAmelCase				( self    :       List[Any]	):
						return ElectraConfig(
						    vocab_size=self.vocab_size     ,hidden_size=self.hidden_size     ,num_hidden_layers=self.num_hidden_layers     ,num_attention_heads=self.num_attention_heads     ,intermediate_size=self.intermediate_size     ,hidden_act=self.hidden_act     ,hidden_dropout_prob=self.hidden_dropout_prob     ,attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,max_position_embeddings=self.max_position_embeddings     ,type_vocab_size=self.type_vocab_size     ,is_decoder=lowercase_     ,initializer_range=self.initializer_range     ,)







					def  __lowerCAmelCase				( self    :       Optional[Any]	):
						(
						    (
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,
						)			:		Optional[Any]						       =			self.prepare_config_and_inputs()

						lowerCAmelCase__			:		Tuple						       =			True
						lowerCAmelCase__			:		str						       =			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
						lowerCAmelCase__			:		List[Any]						       =			ids_tensor([self.batch_size, self.seq_length]     ,vocab_size=2	)

						return (
						    config,
						    input_ids,
						    token_type_ids,
						    input_mask,
						    sequence_labels,
						    token_labels,
						    choice_labels,
						    encoder_hidden_states,
						    encoder_attention_mask,
						)







					def  __lowerCAmelCase				( self    :       List[Any]     ,lowercase_    :       List[str]     ,lowercase_    :       Tuple     ,lowercase_    :       Optional[Any]     ,lowercase_    :       int     ,lowercase_    :       Optional[int]     ,lowercase_    :       Any     ,lowercase_    :       Any     ,lowercase_    :       str     ,):
						lowerCAmelCase__			:		Any						       =			ElectraModel(config=lowercase_	)
						model.to(lowercase_	)
						model.eval()
						lowerCAmelCase__			:		str						       =			model(lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_	)
						lowerCAmelCase__			:		Optional[int]						       =			model(lowercase_     ,token_type_ids=lowercase_	)
						lowerCAmelCase__			:		Optional[Any]						       =			model(lowercase_	)
						self.parent.assertEqual(result.last_hidden_state.shape     ,(self.batch_size, self.seq_length, self.hidden_size)	)







					def  __lowerCAmelCase				( self    :       List[str]     ,lowercase_    :       int     ,lowercase_    :       Union[str, Any]     ,lowercase_    :       Optional[int]     ,lowercase_    :       int     ,lowercase_    :       Union[str, Any]     ,lowercase_    :       int     ,lowercase_    :       List[str]     ,lowercase_    :       int     ,lowercase_    :       int     ,):
						lowerCAmelCase__			:		int						       =			True
						lowerCAmelCase__			:		int						       =			ElectraModel(lowercase_	)
						model.to(lowercase_	)
						model.eval()
						lowerCAmelCase__			:		Optional[Any]						       =			model(
						    lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_     ,encoder_hidden_states=lowercase_     ,encoder_attention_mask=lowercase_     ,)
						lowerCAmelCase__			:		Any						       =			model(
						    lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_     ,encoder_hidden_states=lowercase_     ,)
						lowerCAmelCase__			:		Union[str, Any]						       =			model(lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_	)
						self.parent.assertEqual(result.last_hidden_state.shape     ,(self.batch_size, self.seq_length, self.hidden_size)	)







					def  __lowerCAmelCase				( self    :       Dict     ,lowercase_    :       List[str]     ,lowercase_    :       List[Any]     ,lowercase_    :       Tuple     ,lowercase_    :       List[Any]     ,lowercase_    :       Tuple     ,lowercase_    :       List[str]     ,lowercase_    :       Tuple     ,lowercase_    :       Dict     ,):
						lowerCAmelCase__			:		Dict						       =			ElectraForMaskedLM(config=lowercase_	)
						model.to(lowercase_	)
						model.eval()
						lowerCAmelCase__			:		Any						       =			model(lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_     ,labels=lowercase_	)
						self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.seq_length, self.vocab_size)	)







					def  __lowerCAmelCase				( self    :       Optional[Any]     ,lowercase_    :       Tuple     ,lowercase_    :       Union[str, Any]     ,lowercase_    :       int     ,lowercase_    :       Union[str, Any]     ,lowercase_    :       Dict     ,lowercase_    :       Dict     ,lowercase_    :       Any     ,lowercase_    :       Union[str, Any]     ,lowercase_    :       int     ,):
						lowerCAmelCase__			:		Union[str, Any]						       =			ElectraForCausalLM(config=lowercase_	)
						model.to(lowercase_	)
						model.eval()
						lowerCAmelCase__			:		Tuple						       =			model(lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_     ,labels=lowercase_	)
						self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.seq_length, self.vocab_size)	)







					def  __lowerCAmelCase				( self    :       List[Any]     ,lowercase_    :       Optional[Any]     ,lowercase_    :       List[str]     ,lowercase_    :       List[str]     ,lowercase_    :       Any     ,lowercase_    :       Dict     ,lowercase_    :       str     ,lowercase_    :       List[Any]     ,lowercase_    :       Optional[int]     ,):
						lowerCAmelCase__			:		List[str]						       =			self.num_labels
						lowerCAmelCase__			:		Optional[int]						       =			ElectraForTokenClassification(config=lowercase_	)
						model.to(lowercase_	)
						model.eval()
						lowerCAmelCase__			:		str						       =			model(lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_     ,labels=lowercase_	)
						self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.seq_length, self.num_labels)	)







					def  __lowerCAmelCase				( self    :       str     ,lowercase_    :       Union[str, Any]     ,lowercase_    :       str     ,lowercase_    :       str     ,lowercase_    :       int     ,lowercase_    :       Dict     ,lowercase_    :       Optional[Any]     ,lowercase_    :       Tuple     ,lowercase_    :       List[Any]     ,):
						lowerCAmelCase__			:		Optional[Any]						       =			self.num_labels
						lowerCAmelCase__			:		Union[str, Any]						       =			ElectraForPreTraining(config=lowercase_	)
						model.to(lowercase_	)
						model.eval()
						lowerCAmelCase__			:		Optional[Any]						       =			model(lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_     ,labels=lowercase_	)
						self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.seq_length)	)







					def  __lowerCAmelCase				( self    :       Any     ,lowercase_    :       List[Any]     ,lowercase_    :       str     ,lowercase_    :       int     ,lowercase_    :       List[str]     ,lowercase_    :       Tuple     ,lowercase_    :       List[Any]     ,lowercase_    :       Tuple     ,lowercase_    :       Optional[Any]     ,):
						lowerCAmelCase__			:		str						       =			self.num_labels
						lowerCAmelCase__			:		List[str]						       =			ElectraForSequenceClassification(lowercase_	)
						model.to(lowercase_	)
						model.eval()
						lowerCAmelCase__			:		List[Any]						       =			model(lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_     ,labels=lowercase_	)
						self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.num_labels)	)







					def  __lowerCAmelCase				( self    :       List[str]     ,lowercase_    :       str     ,lowercase_    :       Optional[Any]     ,lowercase_    :       int     ,lowercase_    :       List[Any]     ,lowercase_    :       Dict     ,lowercase_    :       Tuple     ,lowercase_    :       Union[str, Any]     ,lowercase_    :       int     ,):
						lowerCAmelCase__			:		Dict						       =			ElectraForQuestionAnswering(config=lowercase_	)
						model.to(lowercase_	)
						model.eval()
						lowerCAmelCase__			:		Tuple						       =			model(
						    lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_     ,start_positions=lowercase_     ,end_positions=lowercase_     ,)
						self.parent.assertEqual(result.start_logits.shape     ,(self.batch_size, self.seq_length)	)
						self.parent.assertEqual(result.end_logits.shape     ,(self.batch_size, self.seq_length)	)







					def  __lowerCAmelCase				( self    :       str     ,lowercase_    :       int     ,lowercase_    :       str     ,lowercase_    :       Optional[Any]     ,lowercase_    :       str     ,lowercase_    :       Any     ,lowercase_    :       Union[str, Any]     ,lowercase_    :       List[Any]     ,lowercase_    :       Any     ,):
						lowerCAmelCase__			:		int						       =			self.num_choices
						lowerCAmelCase__			:		int						       =			ElectraForMultipleChoice(config=lowercase_	)
						model.to(lowercase_	)
						model.eval()
						lowerCAmelCase__			:		List[Any]						       =			input_ids.unsqueeze(1	).expand(-1     ,self.num_choices     ,-1	).contiguous()
						lowerCAmelCase__			:		Any						       =			token_type_ids.unsqueeze(1	).expand(-1     ,self.num_choices     ,-1	).contiguous()
						lowerCAmelCase__			:		Union[str, Any]						       =			input_mask.unsqueeze(1	).expand(-1     ,self.num_choices     ,-1	).contiguous()
						lowerCAmelCase__			:		Optional[Any]						       =			model(
						    lowercase_     ,attention_mask=lowercase_     ,token_type_ids=lowercase_     ,labels=lowercase_     ,)
						self.parent.assertEqual(result.logits.shape     ,(self.batch_size, self.num_choices)	)







					def  __lowerCAmelCase				( self    :       Tuple	):
						lowerCAmelCase__			:		Optional[int]						       =			self.prepare_config_and_inputs()
						(
						    (
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,(
						    lowerCAmelCase__
						)       ,
						)			:		Tuple						       =			config_and_inputs
						lowerCAmelCase__			:		Tuple						       =			{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
						return config, inputs_dict







@require_torch
class        SCREAMING_SNAKE_CASE      (	a_					,							a_					,							unittest.TestCase ):






					"""simple docstring"""



					lowercase__       				= (
					    (
					        ElectraModel,
					        ElectraForPreTraining,
					        ElectraForMaskedLM,
					        ElectraForCausalLM,
					        ElectraForMultipleChoice,
					        ElectraForTokenClassification,
					        ElectraForSequenceClassification,
					        ElectraForQuestionAnswering,
					    )
					    if is_torch_available()
					    else ()
					)
					lowercase__       				= (
					    {
					        "feature-extraction": ElectraModel,
					        "fill-mask": ElectraForMaskedLM,
					        "question-answering": ElectraForQuestionAnswering,
					        "text-classification": ElectraForSequenceClassification,
					        "text-generation": ElectraForCausalLM,
					        "token-classification": ElectraForTokenClassification,
					        "zero-shot": ElectraForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					lowercase__       				= True







					def  __lowerCAmelCase				( self    :       List[Any]     ,lowercase_    :       Any     ,lowercase_    :       Optional[int]     ,lowercase_    :       int=False	):
						lowerCAmelCase__			:		str						       =			super()._prepare_for_class(lowercase_     ,lowercase_     ,return_labels=lowercase_	)

						if return_labels:
							if model_class in get_values(lowercase_	):
								lowerCAmelCase__			:		Union[str, Any]						       =			torch.zeros(
								    (self.model_tester.batch_size, self.model_tester.seq_length)     ,dtype=torch.long     ,device=lowercase_	)
						return inputs_dict







					def  __lowerCAmelCase				( self    :       str	):
						lowerCAmelCase__			:		List[Any]						       =			ElectraModelTester(self	)
						lowerCAmelCase__			:		List[Any]						       =			ConfigTester(self     ,config_class=lowercase_     ,hidden_size=3_7	)







					def  __lowerCAmelCase				( self    :       List[str]	):
						self.config_tester.run_common_tests()







					def  __lowerCAmelCase				( self    :       Optional[Any]	):
						lowerCAmelCase__			:		List[str]						       =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_electra_model(*lowercase_	)







					def  __lowerCAmelCase				( self    :       Optional[int]	):
						lowerCAmelCase__			:		Dict						       =			self.model_tester.prepare_config_and_inputs_for_decoder()
						self.model_tester.create_and_check_electra_model_as_decoder(*lowercase_	)







					def  __lowerCAmelCase				( self    :       str	):
						lowerCAmelCase__			:		Optional[Any]						       =			self.model_tester.prepare_config_and_inputs()
						for type in ["absolute", "relative_key", "relative_key_query"]:
							lowerCAmelCase__			:		Union[str, Any]						       =			type
							self.model_tester.create_and_check_electra_model(*lowercase_	)







					def  __lowerCAmelCase				( self    :       List[Any]	):
						lowerCAmelCase__			:		Any						       =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_electra_for_masked_lm(*lowercase_	)







					def  __lowerCAmelCase				( self    :       Union[str, Any]	):
						lowerCAmelCase__			:		Dict						       =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_electra_for_token_classification(*lowercase_	)







					def  __lowerCAmelCase				( self    :       Optional[int]	):
						lowerCAmelCase__			:		Tuple						       =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_electra_for_pretraining(*lowercase_	)







					def  __lowerCAmelCase				( self    :       str	):
						lowerCAmelCase__			:		Dict						       =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_electra_for_sequence_classification(*lowercase_	)







					def  __lowerCAmelCase				( self    :       List[Any]	):
						lowerCAmelCase__			:		int						       =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_electra_for_question_answering(*lowercase_	)







					def  __lowerCAmelCase				( self    :       str	):
						lowerCAmelCase__			:		Tuple						       =			self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_electra_for_multiple_choice(*lowercase_	)







					@slow
					def  __lowerCAmelCase				( self    :       Optional[Any]	):
						for model_name in ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
							lowerCAmelCase__			:		Optional[Any]						       =			ElectraModel.from_pretrained(lowercase_	)
							self.assertIsNotNone(lowercase_	)







					def  __lowerCAmelCase				( self    :       int	):
						lowerCAmelCase__			:		int						       =			self.model_tester.prepare_config_and_inputs_for_decoder()
						self.model_tester.create_and_check_electra_for_causal_lm(*lowercase_	)







@require_torch
class        SCREAMING_SNAKE_CASE      (	unittest.TestCase ):






					"""simple docstring"""



					@slow
					def  __lowerCAmelCase				( self    :       List[str]	):
						lowerCAmelCase__			:		Any						       =			ElectraModel.from_pretrained('''google/electra-small-discriminator'''	)
						lowerCAmelCase__			:		Union[str, Any]						       =			torch.tensor([[0, 3_4_5, 2_3_2, 3_2_8, 7_4_0, 1_4_0, 1_6_9_5, 6_9, 6_0_7_8, 1_5_8_8, 2]]	)
						lowerCAmelCase__			:		Tuple						       =			torch.tensor([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]	)
						lowerCAmelCase__			:		Dict						       =			model(lowercase_     ,attention_mask=lowercase_	)[0]
						lowerCAmelCase__			:		int						       =			torch.Size((1, 1_1, 2_5_6)	)
						self.assertEqual(output.shape     ,lowercase_	)
						lowerCAmelCase__			:		Dict						       =			torch.tensor(
						    [[[0.4471, 0.6821, -0.3265], [0.4627, 0.5255, -0.3668], [0.4532, 0.3313, -0.4344]]]	)

						self.assertTrue(torch.allclose(output[:, 1:4, 1:4]     ,lowercase_     ,atol=1E-4	)	)







"""simple docstring"""





from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


__UpperCamelCase						:   str							   =	logging.get_logger(__name__)

__UpperCamelCase						:   Union[str, Any]							   =	{
    '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/config.json''',
    '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/config.json''',
    '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/config.json''',
    '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/config.json''',
    '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/config.json''',
}







class        SCREAMING_SNAKE_CASE      (	a_ ):






					"""simple docstring"""



					lowercase__       				= "gpt2"
					lowercase__       				= ["past_key_values"]
					lowercase__       				= {
					    "hidden_size": "n_embd",
					    "max_position_embeddings": "n_positions",
					    "num_attention_heads": "n_head",
					    "num_hidden_layers": "n_layer",
					}







					def __init__( self    :       List[str]     ,lowercase_    :       Dict=5_0_2_5_7     ,lowercase_    :       Dict=1_0_2_4     ,lowercase_    :       Any=7_6_8     ,lowercase_    :       str=1_2     ,lowercase_    :       int=1_2     ,lowercase_    :       List[Any]=None     ,lowercase_    :       Dict="gelu_new"     ,lowercase_    :       Any=0.1     ,lowercase_    :       Optional[Any]=0.1     ,lowercase_    :       Tuple=0.1     ,lowercase_    :       Dict=1E-5     ,lowercase_    :       List[Any]=0.02     ,lowercase_    :       Tuple="cls_index"     ,lowercase_    :       Optional[int]=True     ,lowercase_    :       Union[str, Any]=None     ,lowercase_    :       Optional[Any]=True     ,lowercase_    :       List[Any]=0.1     ,lowercase_    :       List[str]=True     ,lowercase_    :       int=True     ,lowercase_    :       Optional[Any]=5_0_2_5_6     ,lowercase_    :       Dict=5_0_2_5_6     ,lowercase_    :       Union[str, Any]=False     ,lowercase_    :       Optional[Any]=False     ,**lowercase_    :       Dict     ,):
						lowerCAmelCase__			:		Tuple						       =			vocab_size
						lowerCAmelCase__			:		List[Any]						       =			n_positions
						lowerCAmelCase__			:		Optional[int]						       =			n_embd
						lowerCAmelCase__			:		Tuple						       =			n_layer
						lowerCAmelCase__			:		Optional[Any]						       =			n_head
						lowerCAmelCase__			:		List[Any]						       =			n_inner
						lowerCAmelCase__			:		Any						       =			activation_function
						lowerCAmelCase__			:		Optional[int]						       =			resid_pdrop
						lowerCAmelCase__			:		str						       =			embd_pdrop
						lowerCAmelCase__			:		Tuple						       =			attn_pdrop
						lowerCAmelCase__			:		List[str]						       =			layer_norm_epsilon
						lowerCAmelCase__			:		Union[str, Any]						       =			initializer_range
						lowerCAmelCase__			:		Optional[int]						       =			summary_type
						lowerCAmelCase__			:		Optional[Any]						       =			summary_use_proj
						lowerCAmelCase__			:		List[str]						       =			summary_activation
						lowerCAmelCase__			:		Optional[int]						       =			summary_first_dropout
						lowerCAmelCase__			:		Dict						       =			summary_proj_to_labels
						lowerCAmelCase__			:		Dict						       =			scale_attn_weights
						lowerCAmelCase__			:		Dict						       =			use_cache
						lowerCAmelCase__			:		List[Any]						       =			scale_attn_by_inverse_layer_idx
						lowerCAmelCase__			:		int						       =			reorder_and_upcast_attn

						lowerCAmelCase__			:		List[str]						       =			bos_token_id
						lowerCAmelCase__			:		List[str]						       =			eos_token_id

						super().__init__(bos_token_id=lowercase_     ,eos_token_id=lowercase_     ,**lowercase_	)







class        SCREAMING_SNAKE_CASE      (	a_ ):






					"""simple docstring"""



					def __init__( self    :       Optional[int]     ,lowercase_    :       PretrainedConfig     ,lowercase_    :       str = "default"     ,lowercase_    :       List[PatchingSpec] = None     ,lowercase_    :       bool = False     ,):
						super().__init__(lowercase_     ,task=lowercase_     ,patching_specs=lowercase_     ,use_past=lowercase_	)
						if not getattr(self._config     ,'''pad_token_id'''     ,lowercase_	):
							# TODO: how to do that better?
							lowerCAmelCase__			:		int						       =			0







					@property
					def  __lowerCAmelCase				( self    :       List[str]	):
						lowerCAmelCase__			:		Any						       =			OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}	)
						if self.use_past:
							self.fill_with_past_key_values_(lowercase_     ,direction='''inputs'''	)
							lowerCAmelCase__			:		str						       =			{0: '''batch''', 1: '''past_sequence + sequence'''}
						else:
							lowerCAmelCase__			:		List[str]						       =			{0: '''batch''', 1: '''sequence'''}

						return common_inputs







					@property
					def  __lowerCAmelCase				( self    :       Any	):
						return self._config.n_layer







					@property
					def  __lowerCAmelCase				( self    :       List[str]	):
						return self._config.n_head







					def  __lowerCAmelCase				( self    :       Union[str, Any]     ,lowercase_    :       PreTrainedTokenizer     ,lowercase_    :       int = -1     ,lowercase_    :       int = -1     ,lowercase_    :       bool = False     ,lowercase_    :       Optional[TensorType] = None     ,):
						lowerCAmelCase__			:		Any						       =			super(lowercase_     ,self	).generate_dummy_inputs(
						    lowercase_     ,batch_size=lowercase_     ,seq_length=lowercase_     ,is_pair=lowercase_     ,framework=lowercase_	)

						# We need to order the input in the way they appears in the forward()
						lowerCAmelCase__			:		List[Any]						       =			OrderedDict({'''input_ids''': common_inputs['''input_ids''']}	)

						# Need to add the past_keys
						if self.use_past:
							if not is_torch_available():
								raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''	)
							else:
								import torch

								lowerCAmelCase__       ,lowerCAmelCase__			:		List[Any]						       =			common_inputs['''input_ids'''].shape
								# Not using the same length for past_key_values
								lowerCAmelCase__			:		Dict						       =			seqlen + 2
								lowerCAmelCase__			:		Optional[Any]						       =			(
								    batch,
								    self.num_attention_heads,
								    past_key_values_length,
								    self._config.hidden_size // self.num_attention_heads,
								)
								lowerCAmelCase__			:		Tuple						       =			[
								    (torch.zeros(lowercase_	), torch.zeros(lowercase_	)) for _ in range(self.num_layers	)
								]

						lowerCAmelCase__			:		Any						       =			common_inputs['''attention_mask''']
						if self.use_past:
							lowerCAmelCase__			:		Optional[Any]						       =			ordered_inputs['''attention_mask'''].dtype
							lowerCAmelCase__			:		Union[str, Any]						       =			torch.cat(
							    [ordered_inputs['''attention_mask'''], torch.ones(lowercase_     ,lowercase_     ,dtype=lowercase_	)]     ,dim=1	)

						return ordered_inputs







					@property
					def  __lowerCAmelCase				( self    :       str	):
						return 1_3




