





'''simple docstring'''







import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    MODEL_FOR_PRETRAINING_MAPPING,
	    FunnelBaseModel,
	    FunnelForMaskedLM,
	    FunnelForMultipleChoice,
	    FunnelForPreTraining,
	    FunnelForQuestionAnswering,
	    FunnelForSequenceClassification,
	    FunnelForTokenClassification,
	    FunnelModel,
	)





class 					_UpperCAmelCase							:
		def __init__(						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE=13,__SCREAMING_SNAKE_CASE=7,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=99,__SCREAMING_SNAKE_CASE=[1, 1, 2],__SCREAMING_SNAKE_CASE=1,__SCREAMING_SNAKE_CASE=32,__SCREAMING_SNAKE_CASE=4,__SCREAMING_SNAKE_CASE=8,__SCREAMING_SNAKE_CASE=37,__SCREAMING_SNAKE_CASE="gelu_new",__SCREAMING_SNAKE_CASE=0.1,__SCREAMING_SNAKE_CASE=0.1,__SCREAMING_SNAKE_CASE=0.0,__SCREAMING_SNAKE_CASE=5_12,__SCREAMING_SNAKE_CASE=3,__SCREAMING_SNAKE_CASE=0.02,__SCREAMING_SNAKE_CASE=3,__SCREAMING_SNAKE_CASE=4,__SCREAMING_SNAKE_CASE=None,__SCREAMING_SNAKE_CASE=False,):



					'''simple docstring'''

					__lowerCAmelCase      		=       parent
					__lowerCAmelCase      		=       batch_size
					__lowerCAmelCase      		=       seq_length
					__lowerCAmelCase      		=       is_training
					__lowerCAmelCase      		=       use_input_mask
					__lowerCAmelCase      		=       use_token_type_ids
					__lowerCAmelCase      		=       use_labels
					__lowerCAmelCase      		=       vocab_size
					__lowerCAmelCase      		=       block_sizes
					__lowerCAmelCase      		=       num_decoder_layers
					__lowerCAmelCase      		=       d_model
					__lowerCAmelCase      		=       n_head
					__lowerCAmelCase      		=       d_head
					__lowerCAmelCase      		=       d_inner
					__lowerCAmelCase      		=       hidden_act
					__lowerCAmelCase      		=       hidden_dropout
					__lowerCAmelCase      		=       attention_dropout
					__lowerCAmelCase      		=       activation_dropout
					__lowerCAmelCase      		=       max_position_embeddings
					__lowerCAmelCase      		=       type_vocab_size
					__lowerCAmelCase      		=       2
					__lowerCAmelCase      		=       num_labels
					__lowerCAmelCase      		=       num_choices
					__lowerCAmelCase      		=       scope
					__lowerCAmelCase      		=       initializer_std

					# Used in the tests to check the size of the first attention layer
					__lowerCAmelCase      		=       n_head
					# Used in the tests to check the size of the first hidden state
					__lowerCAmelCase      		=       self.d_model
					# Used in the tests to check the number of output hidden states/attentions
					__lowerCAmelCase      		=       sum(self.block_sizes    ) + (0 if base else self.num_decoder_layers)
					# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
					# the last hidden state of the first block (which is the first hidden state of the decoder).
					if not base:
								__lowerCAmelCase      		=       self.num_hidden_layers + 2
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],self.vocab_size    )

					__lowerCAmelCase      		=       None
					if self.use_input_mask:
								__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],vocab_size=2    )

					__lowerCAmelCase      		=       None
					if self.use_token_type_ids:
								__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],self.type_vocab_size    )

					__lowerCAmelCase      		=       None
					__lowerCAmelCase      		=       None
					__lowerCAmelCase      		=       None
					if self.use_labels:
								__lowerCAmelCase      		=       ids_tensor([self.batch_size],self.type_sequence_label_size    )
								__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],self.num_labels    )
								__lowerCAmelCase      		=       ids_tensor([self.batch_size],self.num_choices    )
								__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],1    )

					__lowerCAmelCase      		=       self.get_config()

					return (
					    config,
					    input_ids,
					    token_type_ids,
					    input_mask,
					    sequence_labels,
					    token_labels,
					    choice_labels,
					    fake_token_labels,
					)
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					return FunnelConfig(
					    vocab_size=self.vocab_size,block_sizes=self.block_sizes,num_decoder_layers=self.num_decoder_layers,d_model=self.d_model,n_head=self.n_head,d_head=self.d_head,d_inner=self.d_inner,hidden_act=self.hidden_act,hidden_dropout=self.hidden_dropout,attention_dropout=self.attention_dropout,activation_dropout=self.activation_dropout,max_position_embeddings=self.max_position_embeddings,type_vocab_size=self.type_vocab_size,initializer_std=self.initializer_std,)
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



					'''simple docstring'''

					__lowerCAmelCase      		=       FunnelModel(config=__SCREAMING_SNAKE_CASE    )
					model.to(__SCREAMING_SNAKE_CASE    )
					model.eval()
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
					self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.d_model)    )

					__lowerCAmelCase      		=       False
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
					self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.d_model)    )

					__lowerCAmelCase      		=       False
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
					self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.d_model)    )
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



					'''simple docstring'''

					__lowerCAmelCase      		=       FunnelBaseModel(config=__SCREAMING_SNAKE_CASE    )
					model.to(__SCREAMING_SNAKE_CASE    )
					model.eval()
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
					self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, 2, self.d_model)    )

					__lowerCAmelCase      		=       False
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
					self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, 3, self.d_model)    )

					__lowerCAmelCase      		=       False
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
					self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, 2, self.d_model)    )
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.num_labels
					__lowerCAmelCase      		=       FunnelForPreTraining(config=__SCREAMING_SNAKE_CASE    )
					model.to(__SCREAMING_SNAKE_CASE    )
					model.eval()
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
					self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length)    )
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



					'''simple docstring'''

					__lowerCAmelCase      		=       FunnelForMaskedLM(config=__SCREAMING_SNAKE_CASE    )
					model.to(__SCREAMING_SNAKE_CASE    )
					model.eval()
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
					self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.vocab_size)    )
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.num_labels
					__lowerCAmelCase      		=       FunnelForSequenceClassification(__SCREAMING_SNAKE_CASE    )
					model.to(__SCREAMING_SNAKE_CASE    )
					model.eval()
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
					self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_labels)    )
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.num_choices
					__lowerCAmelCase      		=       FunnelForMultipleChoice(config=__SCREAMING_SNAKE_CASE    )
					model.to(__SCREAMING_SNAKE_CASE    )
					model.eval()
					__lowerCAmelCase      		=       input_ids.unsqueeze(1    ).expand(-1,self.num_choices,-1    ).contiguous()
					__lowerCAmelCase      		=       token_type_ids.unsqueeze(1    ).expand(-1,self.num_choices,-1    ).contiguous()
					__lowerCAmelCase      		=       input_mask.unsqueeze(1    ).expand(-1,self.num_choices,-1    ).contiguous()
					__lowerCAmelCase      		=       model(
					    __SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE,)
					self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_choices)    )
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.num_labels
					__lowerCAmelCase      		=       FunnelForTokenClassification(config=__SCREAMING_SNAKE_CASE    )
					model.to(__SCREAMING_SNAKE_CASE    )
					model.eval()
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
					self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.num_labels)    )
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



					'''simple docstring'''

					__lowerCAmelCase      		=       FunnelForQuestionAnswering(config=__SCREAMING_SNAKE_CASE    )
					model.to(__SCREAMING_SNAKE_CASE    )
					model.eval()
					__lowerCAmelCase      		=       model(
					    __SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,start_positions=__SCREAMING_SNAKE_CASE,end_positions=__SCREAMING_SNAKE_CASE,)
					self.parent.assertEqual(result.start_logits.shape,(self.batch_size, self.seq_length)    )
					self.parent.assertEqual(result.end_logits.shape,(self.batch_size, self.seq_length)    )







		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.prepare_config_and_inputs()
					(
					    (
					    __lowerCAmelCase
					)		, (
					    __lowerCAmelCase
					)		, (
					    __lowerCAmelCase
					)		, (
					    __lowerCAmelCase
					)		, (
					    __lowerCAmelCase
					)		, (
					    __lowerCAmelCase
					)		, (
					    __lowerCAmelCase
					)		, (
					    __lowerCAmelCase
					)		, 
					)      		=       config_and_inputs
					__lowerCAmelCase      		=       {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
					return config, inputs_dict





@require_torch
class 					_UpperCAmelCase							(		lowerCAmelCase_			,      lowerCAmelCase_			,      unittest.TestCase     ):
		a     :     Optional[int]     						=False
		a     :     Tuple     						=False
		a     :     Union[str, Any]     						=(
		    (
		        FunnelModel,
		        FunnelForMaskedLM,
		        FunnelForPreTraining,
		        FunnelForQuestionAnswering,
		        FunnelForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		a     :     Union[str, Any]     						=(
		    {
		        """feature-extraction""": (FunnelBaseModel, FunnelModel),
		        """fill-mask""": FunnelForMaskedLM,
		        """question-answering""": FunnelForQuestionAnswering,
		        """text-classification""": FunnelForSequenceClassification,
		        """token-classification""": FunnelForTokenClassification,
		        """zero-shot""": FunnelForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE=False    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       super()._prepare_for_class(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,return_labels=__SCREAMING_SNAKE_CASE    )

					if return_labels:
								if model_class in get_values(__SCREAMING_SNAKE_CASE    ):
											__lowerCAmelCase      		=       torch.zeros(
											    (self.model_tester.batch_size, self.model_tester.seq_length),dtype=torch.long,device=__SCREAMING_SNAKE_CASE    )
					return inputs_dict
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       FunnelModelTester(self    )
					__lowerCAmelCase      		=       ConfigTester(self,config_class=__SCREAMING_SNAKE_CASE    )
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					self.config_tester.run_common_tests()
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE    )
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_pretraining(*__SCREAMING_SNAKE_CASE    )
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_masked_lm(*__SCREAMING_SNAKE_CASE    )
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE    )
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE    )







		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					if hasattr(__SCREAMING_SNAKE_CASE,"""weight"""    ) and module.weight is not None:
								module.weight.data.fill_(3    )
					if hasattr(__SCREAMING_SNAKE_CASE,"""bias"""    ) and module.bias is not None:
								module.bias.data.fill_(3    )

					for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
								if hasattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) and getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) is not None:
											__lowerCAmelCase      		=       getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
											weight.data.fill_(3    )





@require_torch
class 					_UpperCAmelCase							(		lowerCAmelCase_			,      unittest.TestCase     ):
		a     :     str     						=False
		a     :     Dict     						=False
		a     :     Any     						=(
		    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
		)
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       FunnelModelTester(self,base=__SCREAMING_SNAKE_CASE    )
					__lowerCAmelCase      		=       ConfigTester(self,config_class=__SCREAMING_SNAKE_CASE    )
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					self.config_tester.run_common_tests()
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_base_model(*__SCREAMING_SNAKE_CASE    )
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE    )
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE    )
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase		, __lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs_for_common()
					__lowerCAmelCase      		=       True

					for model_class in self.all_model_classes:
								if model_class.__name__ == "FunnelBaseModel":
											continue
								__lowerCAmelCase      		=       model_class(__SCREAMING_SNAKE_CASE    )
								model.to(__SCREAMING_SNAKE_CASE    )
								model.train()
								__lowerCAmelCase      		=       self._prepare_for_class(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,return_labels=__SCREAMING_SNAKE_CASE    )
								__lowerCAmelCase      		=       model(**__SCREAMING_SNAKE_CASE    ).loss
								loss.backward()







		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					if hasattr(__SCREAMING_SNAKE_CASE,"""weight"""    ) and module.weight is not None:
								module.weight.data.fill_(3    )
					if hasattr(__SCREAMING_SNAKE_CASE,"""bias"""    ) and module.bias is not None:
								module.bias.data.fill_(3    )

					for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
								if hasattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) and getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) is not None:
											__lowerCAmelCase      		=       getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
											weight.data.fill_(3    )





@require_torch
@require_sentencepiece
@require_tokenizers
class 					_UpperCAmelCase							(		unittest.TestCase     ):
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       13
					__lowerCAmelCase      		=       7
					__lowerCAmelCase      		=       torch.arange(0,batch_size * sequence_length    ).long().reshape(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
					__lowerCAmelCase      		=       [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
					__lowerCAmelCase      		=       torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]    )

					__lowerCAmelCase      		=       FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""    )
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )[0].abs()

					__lowerCAmelCase      		=       torch.tensor(2344.8352    )
					__lowerCAmelCase      		=       torch.tensor(0.8052    )
					self.assertTrue(torch.allclose(output.sum(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
					self.assertTrue(torch.allclose(output.mean(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )

					__lowerCAmelCase      		=       torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]    )
					__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )[0].abs()

					__lowerCAmelCase      		=       torch.tensor(2343.8425    )
					__lowerCAmelCase      		=       torch.tensor(0.8049    )
					self.assertTrue(torch.allclose(output.sum(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
					self.assertTrue(torch.allclose(output.mean(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )







		@slow
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""    )
					__lowerCAmelCase      		=       FunnelModel.from_pretrained("""huggingface/funnel-small"""    )
					__lowerCAmelCase      		=       tokenizer("""Hello! I am the Funnel Transformer model.""",return_tensors="""pt"""    )
					__lowerCAmelCase      		=       model(**__SCREAMING_SNAKE_CASE    )[0]

					__lowerCAmelCase      		=       torch.tensor(235.7246    )
					__lowerCAmelCase      		=       torch.tensor(0.0256    )
					self.assertTrue(torch.allclose(output.sum(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
					self.assertTrue(torch.allclose(output.mean(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )








'''simple docstring'''







def      _lowerCAmelCase		(		lowercase = 1000			)      ->      int:
			__lowerCAmelCase      		=       []
			__lowerCAmelCase      		=       []
			__lowerCAmelCase      		=       3
			__lowerCAmelCase      		=       5
			__lowerCAmelCase      		=       1
			while True:
						__lowerCAmelCase      		=       z * temp
						if result < n:
									zmulti.append(lowercase			)
									temp += 1
						else:
									__lowerCAmelCase      		=       1
									break
			while True:
						__lowerCAmelCase      		=       x * temp
						if result < n:
									xmulti.append(lowercase			)
									temp += 1
						else:
									break
			__lowerCAmelCase      		=       list(set(xmulti + zmulti			)			)
			return sum(lowercase			)


if __name__ == "__main__":
	print(f'{solution() = }')

