



import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    MODEL_FOR_PRETRAINING_MAPPING,
	    FunnelBaseModel,
	    FunnelForMaskedLM,
	    FunnelForMultipleChoice,
	    FunnelForPreTraining,
	    FunnelForQuestionAnswering,
	    FunnelForSequenceClassification,
	    FunnelForTokenClassification,
	    FunnelModel,
	)







class 							A_							:

			def __init__(self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=13,      SCREAMING_SNAKE_CASE_=7,      SCREAMING_SNAKE_CASE_=True,      SCREAMING_SNAKE_CASE_=True,      SCREAMING_SNAKE_CASE_=True,      SCREAMING_SNAKE_CASE_=True,      SCREAMING_SNAKE_CASE_=99,      SCREAMING_SNAKE_CASE_=[1, 1, 2],      SCREAMING_SNAKE_CASE_=1,      SCREAMING_SNAKE_CASE_=32,      SCREAMING_SNAKE_CASE_=4,      SCREAMING_SNAKE_CASE_=8,      SCREAMING_SNAKE_CASE_=37,      SCREAMING_SNAKE_CASE_="gelu_new",      SCREAMING_SNAKE_CASE_=0.1,      SCREAMING_SNAKE_CASE_=0.1,      SCREAMING_SNAKE_CASE_=0.0,      SCREAMING_SNAKE_CASE_=512,      SCREAMING_SNAKE_CASE_=3,      SCREAMING_SNAKE_CASE_=0.0_2,      SCREAMING_SNAKE_CASE_=3,      SCREAMING_SNAKE_CASE_=4,      SCREAMING_SNAKE_CASE_=None,      SCREAMING_SNAKE_CASE_=False,      )					->      Optional[int]:
						UpperCAmelCase_:		str       			=    parent
						UpperCAmelCase_:		Optional[Any]       			=    batch_size
						UpperCAmelCase_:		Optional[Any]       			=    seq_length
						UpperCAmelCase_:		List[str]       			=    is_training
						UpperCAmelCase_:		Any       			=    use_input_mask
						UpperCAmelCase_:		List[Any]       			=    use_token_type_ids
						UpperCAmelCase_:		str       			=    use_labels
						UpperCAmelCase_:		int       			=    vocab_size
						UpperCAmelCase_:		Tuple       			=    block_sizes
						UpperCAmelCase_:		List[str]       			=    num_decoder_layers
						UpperCAmelCase_:		int       			=    d_model
						UpperCAmelCase_:		Union[str, Any]       			=    n_head
						UpperCAmelCase_:		int       			=    d_head
						UpperCAmelCase_:		List[str]       			=    d_inner
						UpperCAmelCase_:		str       			=    hidden_act
						UpperCAmelCase_:		List[Any]       			=    hidden_dropout
						UpperCAmelCase_:		List[Any]       			=    attention_dropout
						UpperCAmelCase_:		Optional[Any]       			=    activation_dropout
						UpperCAmelCase_:		int       			=    max_position_embeddings
						UpperCAmelCase_:		Union[str, Any]       			=    type_vocab_size
						UpperCAmelCase_:		Any       			=    2
						UpperCAmelCase_:		Any       			=    num_labels
						UpperCAmelCase_:		Any       			=    num_choices
						UpperCAmelCase_:		Dict       			=    scope
						UpperCAmelCase_:		Optional[int]       			=    initializer_std

						# Used in the tests to check the size of the first attention layer
						UpperCAmelCase_:		int       			=    n_head
						# Used in the tests to check the size of the first hidden state
						UpperCAmelCase_:		Union[str, Any]       			=    self.d_model
						# Used in the tests to check the number of output hidden states/attentions
						UpperCAmelCase_:		Any       			=    sum(self.block_sizes				) + (0 if base else self.num_decoder_layers)
						# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
						# the last hidden state of the first block (which is the first hidden state of the decoder).
						if not base:
									UpperCAmelCase_:		str       			=    self.num_hidden_layers + 2

			def     __snake_case					(self				)					->      List[str]:
						UpperCAmelCase_:		Tuple       			=    ids_tensor([self.batch_size, self.seq_length],      self.vocab_size				)

						UpperCAmelCase_:		Tuple       			=    None
						if self.use_input_mask:
									UpperCAmelCase_:		Tuple       			=    ids_tensor([self.batch_size, self.seq_length],      vocab_size=2				)

						UpperCAmelCase_:		Optional[Any]       			=    None
						if self.use_token_type_ids:
									UpperCAmelCase_:		Optional[Any]       			=    ids_tensor([self.batch_size, self.seq_length],      self.type_vocab_size				)

						UpperCAmelCase_:		Optional[int]       			=    None
						UpperCAmelCase_:		List[str]       			=    None
						UpperCAmelCase_:		int       			=    None
						if self.use_labels:
									UpperCAmelCase_:		Union[str, Any]       			=    ids_tensor([self.batch_size],      self.type_sequence_label_size				)
									UpperCAmelCase_:		Optional[Any]       			=    ids_tensor([self.batch_size, self.seq_length],      self.num_labels				)
									UpperCAmelCase_:		Union[str, Any]       			=    ids_tensor([self.batch_size],      self.num_choices				)
									UpperCAmelCase_:		Union[str, Any]       			=    ids_tensor([self.batch_size, self.seq_length],      1				)

						UpperCAmelCase_:		List[str]       			=    self.get_config()

						return (
						    config,
						    input_ids,
						    token_type_ids,
						    input_mask,
						    sequence_labels,
						    token_labels,
						    choice_labels,
						    fake_token_labels,
						)

			def     __snake_case					(self				)					->      Optional[int]:
						return FunnelConfig(
						    vocab_size=self.vocab_size,      block_sizes=self.block_sizes,      num_decoder_layers=self.num_decoder_layers,      d_model=self.d_model,      n_head=self.n_head,      d_head=self.d_head,      d_inner=self.d_inner,      hidden_act=self.hidden_act,      hidden_dropout=self.hidden_dropout,      attention_dropout=self.attention_dropout,      activation_dropout=self.activation_dropout,      max_position_embeddings=self.max_position_embeddings,      type_vocab_size=self.type_vocab_size,      initializer_std=self.initializer_std,      )

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      )					->      List[Any]:
						UpperCAmelCase_:		int       			=    FunnelModel(config=SCREAMING_SNAKE_CASE_				)
						model.to(SCREAMING_SNAKE_CASE_				)
						model.eval()
						UpperCAmelCase_:		Any       			=    model(SCREAMING_SNAKE_CASE_,      attention_mask=SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_				)
						UpperCAmelCase_:		int       			=    model(SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_				)
						UpperCAmelCase_:		Optional[Any]       			=    model(SCREAMING_SNAKE_CASE_				)
						self.parent.assertEqual(result.last_hidden_state.shape,      (self.batch_size, self.seq_length, self.d_model)				)

						UpperCAmelCase_:		Optional[int]       			=    False
						UpperCAmelCase_:		List[Any]       			=    model(SCREAMING_SNAKE_CASE_				)
						self.parent.assertEqual(result.last_hidden_state.shape,      (self.batch_size, self.seq_length, self.d_model)				)

						UpperCAmelCase_:		List[Any]       			=    False
						UpperCAmelCase_:		Optional[int]       			=    model(SCREAMING_SNAKE_CASE_				)
						self.parent.assertEqual(result.last_hidden_state.shape,      (self.batch_size, self.seq_length, self.d_model)				)

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      )					->      List[str]:
						UpperCAmelCase_:		str       			=    FunnelBaseModel(config=SCREAMING_SNAKE_CASE_				)
						model.to(SCREAMING_SNAKE_CASE_				)
						model.eval()
						UpperCAmelCase_:		List[str]       			=    model(SCREAMING_SNAKE_CASE_,      attention_mask=SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_				)
						UpperCAmelCase_:		Optional[int]       			=    model(SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_				)
						UpperCAmelCase_:		Any       			=    model(SCREAMING_SNAKE_CASE_				)
						self.parent.assertEqual(result.last_hidden_state.shape,      (self.batch_size, 2, self.d_model)				)

						UpperCAmelCase_:		Dict       			=    False
						UpperCAmelCase_:		int       			=    model(SCREAMING_SNAKE_CASE_				)
						self.parent.assertEqual(result.last_hidden_state.shape,      (self.batch_size, 3, self.d_model)				)

						UpperCAmelCase_:		List[Any]       			=    False
						UpperCAmelCase_:		str       			=    model(SCREAMING_SNAKE_CASE_				)
						self.parent.assertEqual(result.last_hidden_state.shape,      (self.batch_size, 2, self.d_model)				)

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      )					->      Tuple:
						UpperCAmelCase_:		Optional[Any]       			=    self.num_labels
						UpperCAmelCase_:		Optional[int]       			=    FunnelForPreTraining(config=SCREAMING_SNAKE_CASE_				)
						model.to(SCREAMING_SNAKE_CASE_				)
						model.eval()
						UpperCAmelCase_:		Optional[Any]       			=    model(SCREAMING_SNAKE_CASE_,      attention_mask=SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_,      labels=SCREAMING_SNAKE_CASE_				)
						self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.seq_length)				)

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      )					->      Optional[int]:
						UpperCAmelCase_:		Any       			=    FunnelForMaskedLM(config=SCREAMING_SNAKE_CASE_				)
						model.to(SCREAMING_SNAKE_CASE_				)
						model.eval()
						UpperCAmelCase_:		List[Any]       			=    model(SCREAMING_SNAKE_CASE_,      attention_mask=SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_,      labels=SCREAMING_SNAKE_CASE_				)
						self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.seq_length, self.vocab_size)				)

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      )					->      List[str]:
						UpperCAmelCase_:		Union[str, Any]       			=    self.num_labels
						UpperCAmelCase_:		Tuple       			=    FunnelForSequenceClassification(SCREAMING_SNAKE_CASE_				)
						model.to(SCREAMING_SNAKE_CASE_				)
						model.eval()
						UpperCAmelCase_:		Dict       			=    model(SCREAMING_SNAKE_CASE_,      attention_mask=SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_,      labels=SCREAMING_SNAKE_CASE_				)
						self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.num_labels)				)

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      )					->      Dict:
						UpperCAmelCase_:		str       			=    self.num_choices
						UpperCAmelCase_:		Dict       			=    FunnelForMultipleChoice(config=SCREAMING_SNAKE_CASE_				)
						model.to(SCREAMING_SNAKE_CASE_				)
						model.eval()
						UpperCAmelCase_:		Tuple       			=    input_ids.unsqueeze(1				).expand(-1,      self.num_choices,      -1				).contiguous()
						UpperCAmelCase_:		int       			=    token_type_ids.unsqueeze(1				).expand(-1,      self.num_choices,      -1				).contiguous()
						UpperCAmelCase_:		str       			=    input_mask.unsqueeze(1				).expand(-1,      self.num_choices,      -1				).contiguous()
						UpperCAmelCase_:		Any       			=    model(
						    SCREAMING_SNAKE_CASE_,      attention_mask=SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_,      labels=SCREAMING_SNAKE_CASE_,      )
						self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.num_choices)				)

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      )					->      Tuple:
						UpperCAmelCase_:		str       			=    self.num_labels
						UpperCAmelCase_:		List[str]       			=    FunnelForTokenClassification(config=SCREAMING_SNAKE_CASE_				)
						model.to(SCREAMING_SNAKE_CASE_				)
						model.eval()
						UpperCAmelCase_:		Union[str, Any]       			=    model(SCREAMING_SNAKE_CASE_,      attention_mask=SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_,      labels=SCREAMING_SNAKE_CASE_				)
						self.parent.assertEqual(result.logits.shape,      (self.batch_size, self.seq_length, self.num_labels)				)

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      )					->      List[Any]:
						UpperCAmelCase_:		Dict       			=    FunnelForQuestionAnswering(config=SCREAMING_SNAKE_CASE_				)
						model.to(SCREAMING_SNAKE_CASE_				)
						model.eval()
						UpperCAmelCase_:		List[str]       			=    model(
						    SCREAMING_SNAKE_CASE_,      attention_mask=SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_,      start_positions=SCREAMING_SNAKE_CASE_,      end_positions=SCREAMING_SNAKE_CASE_,      )
						self.parent.assertEqual(result.start_logits.shape,      (self.batch_size, self.seq_length)				)
						self.parent.assertEqual(result.end_logits.shape,      (self.batch_size, self.seq_length)				)

			def     __snake_case					(self				)					->      Optional[Any]:
						UpperCAmelCase_:		str       			=    self.prepare_config_and_inputs()
						(
						    UpperCAmelCase_       
						):		Dict       			=    config_and_inputs
						UpperCAmelCase_:		str       			=    {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
						return config, inputs_dict







@require_torch
class 							A_							(      _lowerCAmelCase   ,		_lowerCAmelCase   ,		unittest.TestCase				):
			A					=  False
			A					=  False
			A					=  (
			    (
			        FunnelModel,
			        FunnelForMaskedLM,
			        FunnelForPreTraining,
			        FunnelForQuestionAnswering,
			        FunnelForTokenClassification,
			    )
			    if is_torch_available()
			    else ()
			)
			A					=  (
			    {
			        '''feature-extraction''': (FunnelBaseModel, FunnelModel),
			        '''fill-mask''': FunnelForMaskedLM,
			        '''question-answering''': FunnelForQuestionAnswering,
			        '''text-classification''': FunnelForSequenceClassification,
			        '''token-classification''': FunnelForTokenClassification,
			        '''zero-shot''': FunnelForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_=False				)					->      Union[str, Any]:
						UpperCAmelCase_:		int       			=    super()._prepare_for_class(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      return_labels=SCREAMING_SNAKE_CASE_				)

						if return_labels:
									if model_class in get_values(SCREAMING_SNAKE_CASE_				):
												UpperCAmelCase_:		Union[str, Any]       			=    torch.zeros(
												    (self.model_tester.batch_size, self.model_tester.seq_length),      dtype=torch.long,      device=SCREAMING_SNAKE_CASE_				)
						return inputs_dict

			def     __snake_case					(self				)					->      Any:
						UpperCAmelCase_:		str       			=    FunnelModelTester(self				)
						UpperCAmelCase_:		Optional[Any]       			=    ConfigTester(self,      config_class=SCREAMING_SNAKE_CASE_				)

			def     __snake_case					(self				)					->      Union[str, Any]:
						self.config_tester.run_common_tests()

			def     __snake_case					(self				)					->      Dict:
						UpperCAmelCase_:		Dict       			=    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_				)

			def     __snake_case					(self				)					->      Optional[int]:
						UpperCAmelCase_:		Tuple       			=    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_pretraining(*SCREAMING_SNAKE_CASE_				)

			def     __snake_case					(self				)					->      int:
						UpperCAmelCase_:		str       			=    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE_				)

			def     __snake_case					(self				)					->      List[str]:
						UpperCAmelCase_:		List[Any]       			=    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE_				)

			def     __snake_case					(self				)					->      Union[str, Any]:
						UpperCAmelCase_:		Tuple       			=    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE_				)

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_				)					->      int:
						if hasattr(SCREAMING_SNAKE_CASE_,      """weight"""				) and module.weight is not None:
									module.weight.data.fill_(3				)
						if hasattr(SCREAMING_SNAKE_CASE_,      """bias"""				) and module.bias is not None:
									module.bias.data.fill_(3				)

						for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
									if hasattr(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_				) and getattr(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_				) is not None:
												UpperCAmelCase_:		List[Any]       			=    getattr(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_				)
												weight.data.fill_(3				)







@require_torch
class 							A_							(      _lowerCAmelCase   ,		unittest.TestCase				):
			A					=  False
			A					=  False
			A					=  (
			    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
			)

			def     __snake_case					(self				)					->      List[str]:
						UpperCAmelCase_:		List[Any]       			=    FunnelModelTester(self,      base=SCREAMING_SNAKE_CASE_				)
						UpperCAmelCase_:		Optional[int]       			=    ConfigTester(self,      config_class=SCREAMING_SNAKE_CASE_				)

			def     __snake_case					(self				)					->      int:
						self.config_tester.run_common_tests()

			def     __snake_case					(self				)					->      Optional[int]:
						UpperCAmelCase_:		str       			=    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_base_model(*SCREAMING_SNAKE_CASE_				)

			def     __snake_case					(self				)					->      Union[str, Any]:
						UpperCAmelCase_:		List[Any]       			=    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE_				)

			def     __snake_case					(self				)					->      Union[str, Any]:
						UpperCAmelCase_:		Optional[Any]       			=    self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE_				)

			def     __snake_case					(self				)					->      Optional[Any]:
						UpperCAmelCase_:		List[Any]       			=    self.model_tester.prepare_config_and_inputs_for_common()
						UpperCAmelCase_:		Union[str, Any]       			=    True

						for model_class in self.all_model_classes:
									if model_class.__name__ == "FunnelBaseModel":
												continue
									UpperCAmelCase_:		Dict       			=    model_class(SCREAMING_SNAKE_CASE_				)
									model.to(SCREAMING_SNAKE_CASE_				)
									model.train()
									UpperCAmelCase_:		Optional[Any]       			=    self._prepare_for_class(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_,      return_labels=SCREAMING_SNAKE_CASE_				)
									UpperCAmelCase_:		Dict       			=    model(**SCREAMING_SNAKE_CASE_				).loss
									loss.backward()

			def     __snake_case					(self,      SCREAMING_SNAKE_CASE_				)					->      Optional[Any]:
						if hasattr(SCREAMING_SNAKE_CASE_,      """weight"""				) and module.weight is not None:
									module.weight.data.fill_(3				)
						if hasattr(SCREAMING_SNAKE_CASE_,      """bias"""				) and module.bias is not None:
									module.bias.data.fill_(3				)

						for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
									if hasattr(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_				) and getattr(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_				) is not None:
												UpperCAmelCase_:		str       			=    getattr(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_				)
												weight.data.fill_(3				)







@require_torch
@require_sentencepiece
@require_tokenizers
class 							A_							(      unittest.TestCase				):

			def     __snake_case					(self				)					->      Optional[Any]:
						UpperCAmelCase_:		List[Any]       			=    13
						UpperCAmelCase_:		Optional[int]       			=    7
						UpperCAmelCase_:		List[str]       			=    torch.arange(0,      batch_size * sequence_length				).long().reshape(SCREAMING_SNAKE_CASE_,      SCREAMING_SNAKE_CASE_				)
						UpperCAmelCase_:		str       			=    [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
						UpperCAmelCase_:		Optional[int]       			=    torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]				)

						UpperCAmelCase_:		str       			=    FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""				)
						UpperCAmelCase_:		Tuple       			=    model(SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_				)[0].abs()

						UpperCAmelCase_:		Tuple       			=    torch.tensor(2344.8352				)
						UpperCAmelCase_:		int       			=    torch.tensor(0.8_0_5_2				)
						self.assertTrue(torch.allclose(output.sum(),      SCREAMING_SNAKE_CASE_,      atol=1E-4				)				)
						self.assertTrue(torch.allclose(output.mean(),      SCREAMING_SNAKE_CASE_,      atol=1E-4				)				)

						UpperCAmelCase_:		int       			=    torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]				)
						UpperCAmelCase_:		List[str]       			=    model(SCREAMING_SNAKE_CASE_,      attention_mask=SCREAMING_SNAKE_CASE_,      token_type_ids=SCREAMING_SNAKE_CASE_				)[0].abs()

						UpperCAmelCase_:		Union[str, Any]       			=    torch.tensor(2343.8425				)
						UpperCAmelCase_:		List[Any]       			=    torch.tensor(0.8_0_4_9				)
						self.assertTrue(torch.allclose(output.sum(),      SCREAMING_SNAKE_CASE_,      atol=1E-4				)				)
						self.assertTrue(torch.allclose(output.mean(),      SCREAMING_SNAKE_CASE_,      atol=1E-4				)				)

			@slow
			def     __snake_case					(self				)					->      Union[str, Any]:
						UpperCAmelCase_:		str       			=    FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""				)
						UpperCAmelCase_:		Any       			=    FunnelModel.from_pretrained("""huggingface/funnel-small"""				)
						UpperCAmelCase_:		Optional[int]       			=    tokenizer("""Hello! I am the Funnel Transformer model.""",      return_tensors="""pt"""				)
						UpperCAmelCase_:		str       			=    model(**SCREAMING_SNAKE_CASE_				)[0]

						UpperCAmelCase_:		Optional[Any]       			=    torch.tensor(235.7246				)
						UpperCAmelCase_:		Tuple       			=    torch.tensor(0.0_2_5_6				)
						self.assertTrue(torch.allclose(output.sum(),      SCREAMING_SNAKE_CASE_,      atol=1E-4				)				)
						self.assertTrue(torch.allclose(output.mean(),      SCREAMING_SNAKE_CASE_,      atol=1E-4				)				)











from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a			:     List[str]     = {'configuration_visual_bert': ['VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'VisualBertConfig']}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a			:     List[str]     = [
	    'VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'VisualBertForMultipleChoice',
	    'VisualBertForPreTraining',
	    'VisualBertForQuestionAnswering',
	    'VisualBertForRegionToPhraseAlignment',
	    'VisualBertForVisualReasoning',
	    'VisualBertLayer',
	    'VisualBertModel',
	    'VisualBertPreTrainedModel',
	]


if TYPE_CHECKING:
	from .configuration_visual_bert import VISUAL_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, VisualBertConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_visual_bert import (
		    VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    VisualBertForMultipleChoice,
		    VisualBertForPreTraining,
		    VisualBertForQuestionAnswering,
		    VisualBertForRegionToPhraseAlignment,
		    VisualBertForVisualReasoning,
		    VisualBertLayer,
		    VisualBertModel,
		    VisualBertPreTrainedModel,
		)


else:
	import sys

	a			:     Union[str, Any]     = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






