






import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_PRETRAINING_MAPPING,
		    FunnelBaseModel,
		    FunnelForMaskedLM,
		    FunnelForMultipleChoice,
		    FunnelForPreTraining,
		    FunnelForQuestionAnswering,
		    FunnelForSequenceClassification,
		    FunnelForTokenClassification,
		    FunnelModel,
		)



class 						lowerCamelCase   :







		'''simple docstring'''



		def __init__(							self       ,	_UpperCamelCase       ,	_UpperCamelCase=1_3       ,	_UpperCamelCase=7       ,	_UpperCamelCase=True       ,	_UpperCamelCase=True       ,	_UpperCamelCase=True       ,	_UpperCamelCase=True       ,	_UpperCamelCase=9_9       ,	_UpperCamelCase=[1, 1, 2]       ,	_UpperCamelCase=1       ,	_UpperCamelCase=3_2       ,	_UpperCamelCase=4       ,	_UpperCamelCase=8       ,	_UpperCamelCase=3_7       ,	_UpperCamelCase="gelu_new"       ,	_UpperCamelCase=0.1       ,	_UpperCamelCase=0.1       ,	_UpperCamelCase=0.0       ,	_UpperCamelCase=5_1_2       ,	_UpperCamelCase=3       ,	_UpperCamelCase=0.02       ,	_UpperCamelCase=3       ,	_UpperCamelCase=4       ,	_UpperCamelCase=None       ,	_UpperCamelCase=False       ,	)			->							Tuple:
							UpperCAmelCase_					:   Dict   =		parent
							UpperCAmelCase_					:   Tuple   =		batch_size
							UpperCAmelCase_					:   List[Any]   =		seq_length
							UpperCAmelCase_					:   Dict   =		is_training
							UpperCAmelCase_					:   Dict   =		use_input_mask
							UpperCAmelCase_					:   List[str]   =		use_token_type_ids
							UpperCAmelCase_					:   List[str]   =		use_labels
							UpperCAmelCase_					:   Optional[Any]   =		vocab_size
							UpperCAmelCase_					:   Tuple   =		block_sizes
							UpperCAmelCase_					:   Optional[int]   =		num_decoder_layers
							UpperCAmelCase_					:   Tuple   =		d_model
							UpperCAmelCase_					:   Optional[int]   =		n_head
							UpperCAmelCase_					:   Optional[Any]   =		d_head
							UpperCAmelCase_					:   Optional[Any]   =		d_inner
							UpperCAmelCase_					:   Optional[int]   =		hidden_act
							UpperCAmelCase_					:   int   =		hidden_dropout
							UpperCAmelCase_					:   List[Any]   =		attention_dropout
							UpperCAmelCase_					:   List[Any]   =		activation_dropout
							UpperCAmelCase_					:   List[str]   =		max_position_embeddings
							UpperCAmelCase_					:   int   =		type_vocab_size
							UpperCAmelCase_					:   Union[str, Any]   =		2
							UpperCAmelCase_					:   List[str]   =		num_labels
							UpperCAmelCase_					:   Tuple   =		num_choices
							UpperCAmelCase_					:   Optional[int]   =		scope
							UpperCAmelCase_					:   Optional[Any]   =		initializer_std

							# Used in the tests to check the size of the first attention layer
							UpperCAmelCase_					:   List[Any]   =		n_head
							# Used in the tests to check the size of the first hidden state
							UpperCAmelCase_					:   List[str]   =		self.d_model
							# Used in the tests to check the number of output hidden states/attentions
							UpperCAmelCase_					:   Any   =		sum(self.block_sizes     ) + (0 if base else self.num_decoder_layers)
							# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
							# the last hidden state of the first block (which is the first hidden state of the decoder).
							if not base:
												UpperCAmelCase_					:   Optional[Any]   =		self.num_hidden_layers + 2



		def       __UpperCAmelCase    (							self     )			->							List[str]:
							UpperCAmelCase_					:   Any   =		ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size     )

							UpperCAmelCase_					:   List[Any]   =		None
							if self.use_input_mask:
												UpperCAmelCase_					:   List[str]   =		ids_tensor([self.batch_size, self.seq_length]       ,	vocab_size=2     )

							UpperCAmelCase_					:   Union[str, Any]   =		None
							if self.use_token_type_ids:
												UpperCAmelCase_					:   Optional[Any]   =		ids_tensor([self.batch_size, self.seq_length]       ,	self.type_vocab_size     )

							UpperCAmelCase_					:   str   =		None
							UpperCAmelCase_					:   int   =		None
							UpperCAmelCase_					:   str   =		None
							if self.use_labels:
												UpperCAmelCase_					:   List[Any]   =		ids_tensor([self.batch_size]       ,	self.type_sequence_label_size     )
												UpperCAmelCase_					:   int   =		ids_tensor([self.batch_size, self.seq_length]       ,	self.num_labels     )
												UpperCAmelCase_					:   str   =		ids_tensor([self.batch_size]       ,	self.num_choices     )
												UpperCAmelCase_					:   int   =		ids_tensor([self.batch_size, self.seq_length]       ,	1     )

							UpperCAmelCase_					:   Union[str, Any]   =		self.get_config()

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_mask,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							    fake_token_labels,
							)



		def       __UpperCAmelCase    (							self     )			->							Union[str, Any]:
							return FunnelConfig(
							    vocab_size=self.vocab_size       ,	block_sizes=self.block_sizes       ,	num_decoder_layers=self.num_decoder_layers       ,	d_model=self.d_model       ,	n_head=self.n_head       ,	d_head=self.d_head       ,	d_inner=self.d_inner       ,	hidden_act=self.hidden_act       ,	hidden_dropout=self.hidden_dropout       ,	attention_dropout=self.attention_dropout       ,	activation_dropout=self.activation_dropout       ,	max_position_embeddings=self.max_position_embeddings       ,	type_vocab_size=self.type_vocab_size       ,	initializer_std=self.initializer_std       ,	)



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	)			->							Dict:
							UpperCAmelCase_					:   Optional[int]   =		FunnelModel(config=_UpperCamelCase     )
							model.to(_UpperCamelCase     )
							model.eval()
							UpperCAmelCase_					:   Dict   =		model(_UpperCamelCase       ,	attention_mask=_UpperCamelCase       ,	token_type_ids=_UpperCamelCase     )
							UpperCAmelCase_					:   str   =		model(_UpperCamelCase       ,	token_type_ids=_UpperCamelCase     )
							UpperCAmelCase_					:   List[str]   =		model(_UpperCamelCase     )
							self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.d_model)     )

							UpperCAmelCase_					:   Any   =		False
							UpperCAmelCase_					:   List[Any]   =		model(_UpperCamelCase     )
							self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.d_model)     )

							UpperCAmelCase_					:   Any   =		False
							UpperCAmelCase_					:   Tuple   =		model(_UpperCamelCase     )
							self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.d_model)     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	)			->							Dict:
							UpperCAmelCase_					:   Dict   =		FunnelBaseModel(config=_UpperCamelCase     )
							model.to(_UpperCamelCase     )
							model.eval()
							UpperCAmelCase_					:   int   =		model(_UpperCamelCase       ,	attention_mask=_UpperCamelCase       ,	token_type_ids=_UpperCamelCase     )
							UpperCAmelCase_					:   Optional[int]   =		model(_UpperCamelCase       ,	token_type_ids=_UpperCamelCase     )
							UpperCAmelCase_					:   Optional[int]   =		model(_UpperCamelCase     )
							self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, 2, self.d_model)     )

							UpperCAmelCase_					:   List[Any]   =		False
							UpperCAmelCase_					:   Union[str, Any]   =		model(_UpperCamelCase     )
							self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, 3, self.d_model)     )

							UpperCAmelCase_					:   List[Any]   =		False
							UpperCAmelCase_					:   List[Any]   =		model(_UpperCamelCase     )
							self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, 2, self.d_model)     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	)			->							Dict:
							UpperCAmelCase_					:   Optional[Any]   =		self.num_labels
							UpperCAmelCase_					:   Optional[int]   =		FunnelForPreTraining(config=_UpperCamelCase     )
							model.to(_UpperCamelCase     )
							model.eval()
							UpperCAmelCase_					:   str   =		model(_UpperCamelCase       ,	attention_mask=_UpperCamelCase       ,	token_type_ids=_UpperCamelCase       ,	labels=_UpperCamelCase     )
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length)     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	)			->							Optional[Any]:
							UpperCAmelCase_					:   Dict   =		FunnelForMaskedLM(config=_UpperCamelCase     )
							model.to(_UpperCamelCase     )
							model.eval()
							UpperCAmelCase_					:   int   =		model(_UpperCamelCase       ,	attention_mask=_UpperCamelCase       ,	token_type_ids=_UpperCamelCase       ,	labels=_UpperCamelCase     )
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	)			->							List[str]:
							UpperCAmelCase_					:   Tuple   =		self.num_labels
							UpperCAmelCase_					:   int   =		FunnelForSequenceClassification(_UpperCamelCase     )
							model.to(_UpperCamelCase     )
							model.eval()
							UpperCAmelCase_					:   Optional[Any]   =		model(_UpperCamelCase       ,	attention_mask=_UpperCamelCase       ,	token_type_ids=_UpperCamelCase       ,	labels=_UpperCamelCase     )
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.num_labels)     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	)			->							Union[str, Any]:
							UpperCAmelCase_					:   Tuple   =		self.num_choices
							UpperCAmelCase_					:   Tuple   =		FunnelForMultipleChoice(config=_UpperCamelCase     )
							model.to(_UpperCamelCase     )
							model.eval()
							UpperCAmelCase_					:   Tuple   =		input_ids.unsqueeze(1     ).expand(-1       ,	self.num_choices       ,	-1     ).contiguous()
							UpperCAmelCase_					:   List[Any]   =		token_type_ids.unsqueeze(1     ).expand(-1       ,	self.num_choices       ,	-1     ).contiguous()
							UpperCAmelCase_					:   Optional[int]   =		input_mask.unsqueeze(1     ).expand(-1       ,	self.num_choices       ,	-1     ).contiguous()
							UpperCAmelCase_					:   Optional[int]   =		model(
							    _UpperCamelCase       ,	attention_mask=_UpperCamelCase       ,	token_type_ids=_UpperCamelCase       ,	labels=_UpperCamelCase       ,	)
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.num_choices)     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	)			->							Optional[Any]:
							UpperCAmelCase_					:   Union[str, Any]   =		self.num_labels
							UpperCAmelCase_					:   str   =		FunnelForTokenClassification(config=_UpperCamelCase     )
							model.to(_UpperCamelCase     )
							model.eval()
							UpperCAmelCase_					:   Any   =		model(_UpperCamelCase       ,	attention_mask=_UpperCamelCase       ,	token_type_ids=_UpperCamelCase       ,	labels=_UpperCamelCase     )
							self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.num_labels)     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	)			->							Optional[int]:
							UpperCAmelCase_					:   Optional[int]   =		FunnelForQuestionAnswering(config=_UpperCamelCase     )
							model.to(_UpperCamelCase     )
							model.eval()
							UpperCAmelCase_					:   int   =		model(
							    _UpperCamelCase       ,	attention_mask=_UpperCamelCase       ,	token_type_ids=_UpperCamelCase       ,	start_positions=_UpperCamelCase       ,	end_positions=_UpperCamelCase       ,	)
							self.parent.assertEqual(result.start_logits.shape       ,	(self.batch_size, self.seq_length)     )
							self.parent.assertEqual(result.end_logits.shape       ,	(self.batch_size, self.seq_length)     )



		def       __UpperCAmelCase    (							self     )			->							Union[str, Any]:
							UpperCAmelCase_					:   Optional[int]   =		self.prepare_config_and_inputs()
							(
							    (
							    UpperCAmelCase_
							) ,						(
							    UpperCAmelCase_
							) ,						(
							    UpperCAmelCase_
							) ,						(
							    UpperCAmelCase_
							) ,						(
							    UpperCAmelCase_
							) ,						(
							    UpperCAmelCase_
							) ,						(
							    UpperCAmelCase_
							) ,						(
							    UpperCAmelCase_
							) ,						
							)					:   Union[str, Any]   =		config_and_inputs
							UpperCAmelCase_					:   List[Any]   =		{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict





@require_torch
class 						lowerCamelCase   (_snake_case				,						_snake_case				,						unittest.TestCase    ):







		'''simple docstring'''



		_snake_case					:					int 		=						False
		_snake_case					:					Dict 		=						False
		_snake_case					:					Dict 		=						(
		    (
		        FunnelModel,
		        FunnelForMaskedLM,
		        FunnelForPreTraining,
		        FunnelForQuestionAnswering,
		        FunnelForTokenClassification,
		    )
		    if is_torch_available()
		    else ()
		)
		_snake_case					:					Optional[Any] 		=						(
		    {
		        '''feature-extraction''': (FunnelBaseModel, FunnelModel),
		        '''fill-mask''': FunnelForMaskedLM,
		        '''question-answering''': FunnelForQuestionAnswering,
		        '''text-classification''': FunnelForSequenceClassification,
		        '''token-classification''': FunnelForTokenClassification,
		        '''zero-shot''': FunnelForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase       ,	_UpperCamelCase       ,	_UpperCamelCase=False     )			->							Union[str, Any]:
							UpperCAmelCase_					:   Tuple   =		super()._prepare_for_class(_UpperCamelCase       ,	_UpperCamelCase       ,	return_labels=_UpperCamelCase     )

							if return_labels:
												if model_class in get_values(_UpperCamelCase     ):
																	UpperCAmelCase_					:   List[Any]   =		torch.zeros(
																	    (self.model_tester.batch_size, self.model_tester.seq_length)       ,	dtype=torch.long       ,	device=_UpperCamelCase     )
							return inputs_dict



		def       __UpperCAmelCase    (							self     )			->							str:
							UpperCAmelCase_					:   List[Any]   =		FunnelModelTester(self     )
							UpperCAmelCase_					:   int   =		ConfigTester(self       ,	config_class=_UpperCamelCase     )



		def       __UpperCAmelCase    (							self     )			->							str:
							self.config_tester.run_common_tests()



		def       __UpperCAmelCase    (							self     )			->							Tuple:
							UpperCAmelCase_					:   str   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*_UpperCamelCase     )



		def       __UpperCAmelCase    (							self     )			->							Tuple:
							UpperCAmelCase_					:   List[Any]   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_pretraining(*_UpperCamelCase     )



		def       __UpperCAmelCase    (							self     )			->							List[Any]:
							UpperCAmelCase_					:   Any   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*_UpperCamelCase     )



		def       __UpperCAmelCase    (							self     )			->							int:
							UpperCAmelCase_					:   Union[str, Any]   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*_UpperCamelCase     )



		def       __UpperCAmelCase    (							self     )			->							List[Any]:
							UpperCAmelCase_					:   Optional[int]   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*_UpperCamelCase     )



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase     )			->							Optional[Any]:
							if hasattr(_UpperCamelCase       ,	'weight'     ) and module.weight is not None:
												module.weight.data.fill_(3     )
							if hasattr(_UpperCamelCase       ,	'bias'     ) and module.bias is not None:
												module.bias.data.fill_(3     )

							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
												if hasattr(_UpperCamelCase       ,	_UpperCamelCase     ) and getattr(_UpperCamelCase       ,	_UpperCamelCase     ) is not None:
																	UpperCAmelCase_					:   int   =		getattr(_UpperCamelCase       ,	_UpperCamelCase     )
																	weight.data.fill_(3     )





@require_torch
class 						lowerCamelCase   (_snake_case				,						unittest.TestCase    ):







		'''simple docstring'''



		_snake_case					:					Dict 		=						False
		_snake_case					:					Tuple 		=						False
		_snake_case					:					Any 		=						(
		    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
		)



		def       __UpperCAmelCase    (							self     )			->							List[str]:
							UpperCAmelCase_					:   Dict   =		FunnelModelTester(self       ,	base=_UpperCamelCase     )
							UpperCAmelCase_					:   Optional[int]   =		ConfigTester(self       ,	config_class=_UpperCamelCase     )



		def       __UpperCAmelCase    (							self     )			->							List[str]:
							self.config_tester.run_common_tests()



		def       __UpperCAmelCase    (							self     )			->							str:
							UpperCAmelCase_					:   List[Any]   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_base_model(*_UpperCamelCase     )



		def       __UpperCAmelCase    (							self     )			->							Optional[int]:
							UpperCAmelCase_					:   str   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*_UpperCamelCase     )



		def       __UpperCAmelCase    (							self     )			->							Any:
							UpperCAmelCase_					:   Tuple   =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*_UpperCamelCase     )



		def       __UpperCAmelCase    (							self     )			->							Any:
							UpperCAmelCase_ ,						UpperCAmelCase_					:   Optional[int]   =		self.model_tester.prepare_config_and_inputs_for_common()
							UpperCAmelCase_					:   Optional[Any]   =		True

							for model_class in self.all_model_classes:
												if model_class.__name__ == "FunnelBaseModel":
																	continue
												UpperCAmelCase_					:   Dict   =		model_class(_UpperCamelCase     )
												model.to(_UpperCamelCase     )
												model.train()
												UpperCAmelCase_					:   Any   =		self._prepare_for_class(_UpperCamelCase       ,	_UpperCamelCase       ,	return_labels=_UpperCamelCase     )
												UpperCAmelCase_					:   Union[str, Any]   =		model(**_UpperCamelCase     ).loss
												loss.backward()



		def       __UpperCAmelCase    (							self       ,	_UpperCamelCase     )			->							Dict:
							if hasattr(_UpperCamelCase       ,	'weight'     ) and module.weight is not None:
												module.weight.data.fill_(3     )
							if hasattr(_UpperCamelCase       ,	'bias'     ) and module.bias is not None:
												module.bias.data.fill_(3     )

							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
												if hasattr(_UpperCamelCase       ,	_UpperCamelCase     ) and getattr(_UpperCamelCase       ,	_UpperCamelCase     ) is not None:
																	UpperCAmelCase_					:   Union[str, Any]   =		getattr(_UpperCamelCase       ,	_UpperCamelCase     )
																	weight.data.fill_(3     )





@require_torch
@require_sentencepiece
@require_tokenizers
class 						lowerCamelCase   (unittest.TestCase    ):







		'''simple docstring'''



		def       __UpperCAmelCase    (							self     )			->							str:
							UpperCAmelCase_					:   List[str]   =		1_3
							UpperCAmelCase_					:   str   =		7
							UpperCAmelCase_					:   int   =		torch.arange(0       ,	batch_size * sequence_length     ).long().reshape(_UpperCamelCase       ,	_UpperCamelCase     )
							UpperCAmelCase_					:   Tuple   =		[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
							UpperCAmelCase_					:   int   =		torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]     )

							UpperCAmelCase_					:   Union[str, Any]   =		FunnelModel.from_pretrained('sgugger/funnel-random-tiny'     )
							UpperCAmelCase_					:   Union[str, Any]   =		model(_UpperCamelCase       ,	token_type_ids=_UpperCamelCase     )[0].abs()

							UpperCAmelCase_					:   Optional[Any]   =		torch.tensor(23_44.83_52     )
							UpperCAmelCase_					:   Optional[Any]   =		torch.tensor(0.80_52     )
							self.assertTrue(torch.allclose(output.sum()       ,	_UpperCamelCase       ,	atol=1E-4     )     )
							self.assertTrue(torch.allclose(output.mean()       ,	_UpperCamelCase       ,	atol=1E-4     )     )

							UpperCAmelCase_					:   Union[str, Any]   =		torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]     )
							UpperCAmelCase_					:   Tuple   =		model(_UpperCamelCase       ,	attention_mask=_UpperCamelCase       ,	token_type_ids=_UpperCamelCase     )[0].abs()

							UpperCAmelCase_					:   Optional[int]   =		torch.tensor(23_43.84_25     )
							UpperCAmelCase_					:   List[str]   =		torch.tensor(0.80_49     )
							self.assertTrue(torch.allclose(output.sum()       ,	_UpperCamelCase       ,	atol=1E-4     )     )
							self.assertTrue(torch.allclose(output.mean()       ,	_UpperCamelCase       ,	atol=1E-4     )     )



		@slow
		def       __UpperCAmelCase    (							self     )			->							List[str]:
							UpperCAmelCase_					:   List[str]   =		FunnelTokenizer.from_pretrained('huggingface/funnel-small'     )
							UpperCAmelCase_					:   Any   =		FunnelModel.from_pretrained('huggingface/funnel-small'     )
							UpperCAmelCase_					:   Tuple   =		tokenizer('Hello! I am the Funnel Transformer model.'       ,	return_tensors='pt'     )
							UpperCAmelCase_					:   Any   =		model(**_UpperCamelCase     )[0]

							UpperCAmelCase_					:   int   =		torch.tensor(2_35.72_46     )
							UpperCAmelCase_					:   Optional[Any]   =		torch.tensor(0.02_56     )
							self.assertTrue(torch.allclose(output.sum()       ,	_UpperCamelCase       ,	atol=1E-4     )     )
							self.assertTrue(torch.allclose(output.mean()       ,	_UpperCamelCase       ,	atol=1E-4     )     )














from typing import TYPE_CHECKING

from ...utils import _LazyModule


__UpperCAmelCase										=		{'tokenization_phobert': ['PhobertTokenizer']}


if TYPE_CHECKING:
		from .tokenization_phobert import PhobertTokenizer

else:
		import sys

		__UpperCAmelCase										=		_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)






