





'''simple docstring'''







import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    MODEL_FOR_PRETRAINING_MAPPING,
	    FunnelBaseModel,
	    FunnelForMaskedLM,
	    FunnelForMultipleChoice,
	    FunnelForPreTraining,
	    FunnelForQuestionAnswering,
	    FunnelForSequenceClassification,
	    FunnelForTokenClassification,
	    FunnelModel,
	)





class 					_UpperCAmelCase							:
				def __init__(						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE=13,__SCREAMING_SNAKE_CASE=7,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=True,__SCREAMING_SNAKE_CASE=99,__SCREAMING_SNAKE_CASE=[1, 1, 2],__SCREAMING_SNAKE_CASE=1,__SCREAMING_SNAKE_CASE=32,__SCREAMING_SNAKE_CASE=4,__SCREAMING_SNAKE_CASE=8,__SCREAMING_SNAKE_CASE=37,__SCREAMING_SNAKE_CASE="gelu_new",__SCREAMING_SNAKE_CASE=0.1,__SCREAMING_SNAKE_CASE=0.1,__SCREAMING_SNAKE_CASE=0.0,__SCREAMING_SNAKE_CASE=5_12,__SCREAMING_SNAKE_CASE=3,__SCREAMING_SNAKE_CASE=0.02,__SCREAMING_SNAKE_CASE=3,__SCREAMING_SNAKE_CASE=4,__SCREAMING_SNAKE_CASE=None,__SCREAMING_SNAKE_CASE=False,):



							'''simple docstring'''

							__lowerCAmelCase      		=       parent
							__lowerCAmelCase      		=       batch_size
							__lowerCAmelCase      		=       seq_length
							__lowerCAmelCase      		=       is_training
							__lowerCAmelCase      		=       use_input_mask
							__lowerCAmelCase      		=       use_token_type_ids
							__lowerCAmelCase      		=       use_labels
							__lowerCAmelCase      		=       vocab_size
							__lowerCAmelCase      		=       block_sizes
							__lowerCAmelCase      		=       num_decoder_layers
							__lowerCAmelCase      		=       d_model
							__lowerCAmelCase      		=       n_head
							__lowerCAmelCase      		=       d_head
							__lowerCAmelCase      		=       d_inner
							__lowerCAmelCase      		=       hidden_act
							__lowerCAmelCase      		=       hidden_dropout
							__lowerCAmelCase      		=       attention_dropout
							__lowerCAmelCase      		=       activation_dropout
							__lowerCAmelCase      		=       max_position_embeddings
							__lowerCAmelCase      		=       type_vocab_size
							__lowerCAmelCase      		=       2
							__lowerCAmelCase      		=       num_labels
							__lowerCAmelCase      		=       num_choices
							__lowerCAmelCase      		=       scope
							__lowerCAmelCase      		=       initializer_std

							# Used in the tests to check the size of the first attention layer
							__lowerCAmelCase      		=       n_head
							# Used in the tests to check the size of the first hidden state
							__lowerCAmelCase      		=       self.d_model
							# Used in the tests to check the number of output hidden states/attentions
							__lowerCAmelCase      		=       sum(self.block_sizes    ) + (0 if base else self.num_decoder_layers)
							# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
							# the last hidden state of the first block (which is the first hidden state of the decoder).
							if not base:
										__lowerCAmelCase      		=       self.num_hidden_layers + 2
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],self.vocab_size    )

							__lowerCAmelCase      		=       None
							if self.use_input_mask:
										__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],vocab_size=2    )

							__lowerCAmelCase      		=       None
							if self.use_token_type_ids:
										__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],self.type_vocab_size    )

							__lowerCAmelCase      		=       None
							__lowerCAmelCase      		=       None
							__lowerCAmelCase      		=       None
							if self.use_labels:
										__lowerCAmelCase      		=       ids_tensor([self.batch_size],self.type_sequence_label_size    )
										__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],self.num_labels    )
										__lowerCAmelCase      		=       ids_tensor([self.batch_size],self.num_choices    )
										__lowerCAmelCase      		=       ids_tensor([self.batch_size, self.seq_length],1    )

							__lowerCAmelCase      		=       self.get_config()

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_mask,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							    fake_token_labels,
							)
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							return FunnelConfig(
							    vocab_size=self.vocab_size,block_sizes=self.block_sizes,num_decoder_layers=self.num_decoder_layers,d_model=self.d_model,n_head=self.n_head,d_head=self.d_head,d_inner=self.d_inner,hidden_act=self.hidden_act,hidden_dropout=self.hidden_dropout,attention_dropout=self.attention_dropout,activation_dropout=self.activation_dropout,max_position_embeddings=self.max_position_embeddings,type_vocab_size=self.type_vocab_size,initializer_std=self.initializer_std,)
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



							'''simple docstring'''

							__lowerCAmelCase      		=       FunnelModel(config=__SCREAMING_SNAKE_CASE    )
							model.to(__SCREAMING_SNAKE_CASE    )
							model.eval()
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.d_model)    )

							__lowerCAmelCase      		=       False
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.d_model)    )

							__lowerCAmelCase      		=       False
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, self.seq_length, self.d_model)    )
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



							'''simple docstring'''

							__lowerCAmelCase      		=       FunnelBaseModel(config=__SCREAMING_SNAKE_CASE    )
							model.to(__SCREAMING_SNAKE_CASE    )
							model.eval()
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, 2, self.d_model)    )

							__lowerCAmelCase      		=       False
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, 3, self.d_model)    )

							__lowerCAmelCase      		=       False
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE    )
							self.parent.assertEqual(result.last_hidden_state.shape,(self.batch_size, 2, self.d_model)    )
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.num_labels
							__lowerCAmelCase      		=       FunnelForPreTraining(config=__SCREAMING_SNAKE_CASE    )
							model.to(__SCREAMING_SNAKE_CASE    )
							model.eval()
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
							self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length)    )
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



							'''simple docstring'''

							__lowerCAmelCase      		=       FunnelForMaskedLM(config=__SCREAMING_SNAKE_CASE    )
							model.to(__SCREAMING_SNAKE_CASE    )
							model.eval()
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
							self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.vocab_size)    )
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.num_labels
							__lowerCAmelCase      		=       FunnelForSequenceClassification(__SCREAMING_SNAKE_CASE    )
							model.to(__SCREAMING_SNAKE_CASE    )
							model.eval()
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
							self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_labels)    )
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.num_choices
							__lowerCAmelCase      		=       FunnelForMultipleChoice(config=__SCREAMING_SNAKE_CASE    )
							model.to(__SCREAMING_SNAKE_CASE    )
							model.eval()
							__lowerCAmelCase      		=       input_ids.unsqueeze(1    ).expand(-1,self.num_choices,-1    ).contiguous()
							__lowerCAmelCase      		=       token_type_ids.unsqueeze(1    ).expand(-1,self.num_choices,-1    ).contiguous()
							__lowerCAmelCase      		=       input_mask.unsqueeze(1    ).expand(-1,self.num_choices,-1    ).contiguous()
							__lowerCAmelCase      		=       model(
							    __SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE,)
							self.parent.assertEqual(result.logits.shape,(self.batch_size, self.num_choices)    )
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.num_labels
							__lowerCAmelCase      		=       FunnelForTokenClassification(config=__SCREAMING_SNAKE_CASE    )
							model.to(__SCREAMING_SNAKE_CASE    )
							model.eval()
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,labels=__SCREAMING_SNAKE_CASE    )
							self.parent.assertEqual(result.logits.shape,(self.batch_size, self.seq_length, self.num_labels)    )
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,):



							'''simple docstring'''

							__lowerCAmelCase      		=       FunnelForQuestionAnswering(config=__SCREAMING_SNAKE_CASE    )
							model.to(__SCREAMING_SNAKE_CASE    )
							model.eval()
							__lowerCAmelCase      		=       model(
							    __SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE,start_positions=__SCREAMING_SNAKE_CASE,end_positions=__SCREAMING_SNAKE_CASE,)
							self.parent.assertEqual(result.start_logits.shape,(self.batch_size, self.seq_length)    )
							self.parent.assertEqual(result.end_logits.shape,(self.batch_size, self.seq_length)    )







				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.prepare_config_and_inputs()
							(
							    (
							    __lowerCAmelCase 
							)		, (
							    __lowerCAmelCase 
							)		, (
							    __lowerCAmelCase 
							)		, (
							    __lowerCAmelCase 
							)		, (
							    __lowerCAmelCase 
							)		, (
							    __lowerCAmelCase 
							)		, (
							    __lowerCAmelCase 
							)		, (
							    __lowerCAmelCase 
							)		,  
							)      		=       config_and_inputs
							__lowerCAmelCase      		=       {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
							return config, inputs_dict





@require_torch
class 					_UpperCAmelCase							(		lowerCAmelCase_			,      lowerCAmelCase_			,      unittest.TestCase     ):
				a     :     Optional[int]     						=False
				a     :     Tuple     						=False
				a     :     Union[str, Any]     						=(
				    (
				        FunnelModel,
				        FunnelForMaskedLM,
				        FunnelForPreTraining,
				        FunnelForQuestionAnswering,
				        FunnelForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				a     :     Union[str, Any]     						=(
				    {
				        """feature-extraction""": (FunnelBaseModel, FunnelModel),
				        """fill-mask""": FunnelForMaskedLM,
				        """question-answering""": FunnelForQuestionAnswering,
				        """text-classification""": FunnelForSequenceClassification,
				        """token-classification""": FunnelForTokenClassification,
				        """zero-shot""": FunnelForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE=False    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       super()._prepare_for_class(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,return_labels=__SCREAMING_SNAKE_CASE    )

							if return_labels:
										if model_class in get_values(__SCREAMING_SNAKE_CASE    ):
													__lowerCAmelCase      		=       torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.seq_length),dtype=torch.long,device=__SCREAMING_SNAKE_CASE    )
							return inputs_dict
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       FunnelModelTester(self    )
							__lowerCAmelCase      		=       ConfigTester(self,config_class=__SCREAMING_SNAKE_CASE    )
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							self.config_tester.run_common_tests()
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE    )
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_pretraining(*__SCREAMING_SNAKE_CASE    )
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*__SCREAMING_SNAKE_CASE    )
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE    )
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE    )







				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



							'''simple docstring'''

							if hasattr(__SCREAMING_SNAKE_CASE,"""weight"""    ) and module.weight is not None:
										module.weight.data.fill_(3    )
							if hasattr(__SCREAMING_SNAKE_CASE,"""bias"""    ) and module.bias is not None:
										module.bias.data.fill_(3    )

							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
										if hasattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) and getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) is not None:
													__lowerCAmelCase      		=       getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
													weight.data.fill_(3    )





@require_torch
class 					_UpperCAmelCase							(		lowerCAmelCase_			,      unittest.TestCase     ):
				a     :     str     						=False
				a     :     Dict     						=False
				a     :     Any     						=(
				    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
				)
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       FunnelModelTester(self,base=__SCREAMING_SNAKE_CASE    )
							__lowerCAmelCase      		=       ConfigTester(self,config_class=__SCREAMING_SNAKE_CASE    )
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							self.config_tester.run_common_tests()
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_base_model(*__SCREAMING_SNAKE_CASE    )
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE    )
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE    )
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase		, __lowerCAmelCase      		=       self.model_tester.prepare_config_and_inputs_for_common()
							__lowerCAmelCase      		=       True

							for model_class in self.all_model_classes:
										if model_class.__name__ == "FunnelBaseModel":
													continue
										__lowerCAmelCase      		=       model_class(__SCREAMING_SNAKE_CASE    )
										model.to(__SCREAMING_SNAKE_CASE    )
										model.train()
										__lowerCAmelCase      		=       self._prepare_for_class(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE,return_labels=__SCREAMING_SNAKE_CASE    )
										__lowerCAmelCase      		=       model(**__SCREAMING_SNAKE_CASE    ).loss
										loss.backward()







				def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



							'''simple docstring'''

							if hasattr(__SCREAMING_SNAKE_CASE,"""weight"""    ) and module.weight is not None:
										module.weight.data.fill_(3    )
							if hasattr(__SCREAMING_SNAKE_CASE,"""bias"""    ) and module.bias is not None:
										module.bias.data.fill_(3    )

							for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
										if hasattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) and getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ) is not None:
													__lowerCAmelCase      		=       getattr(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
													weight.data.fill_(3    )





@require_torch
@require_sentencepiece
@require_tokenizers
class 					_UpperCAmelCase							(		unittest.TestCase     ):
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       13
							__lowerCAmelCase      		=       7
							__lowerCAmelCase      		=       torch.arange(0,batch_size * sequence_length    ).long().reshape(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
							__lowerCAmelCase      		=       [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
							__lowerCAmelCase      		=       torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]    )

							__lowerCAmelCase      		=       FunnelModel.from_pretrained("""sgugger/funnel-random-tiny"""    )
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )[0].abs()

							__lowerCAmelCase      		=       torch.tensor(2344.8352    )
							__lowerCAmelCase      		=       torch.tensor(0.8052    )
							self.assertTrue(torch.allclose(output.sum(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
							self.assertTrue(torch.allclose(output.mean(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )

							__lowerCAmelCase      		=       torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]    )
							__lowerCAmelCase      		=       model(__SCREAMING_SNAKE_CASE,attention_mask=__SCREAMING_SNAKE_CASE,token_type_ids=__SCREAMING_SNAKE_CASE    )[0].abs()

							__lowerCAmelCase      		=       torch.tensor(2343.8425    )
							__lowerCAmelCase      		=       torch.tensor(0.8049    )
							self.assertTrue(torch.allclose(output.sum(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
							self.assertTrue(torch.allclose(output.mean(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )







				@slow
				def 						lowerCamelCase__    (						self    ):



							'''simple docstring'''

							__lowerCAmelCase      		=       FunnelTokenizer.from_pretrained("""huggingface/funnel-small"""    )
							__lowerCAmelCase      		=       FunnelModel.from_pretrained("""huggingface/funnel-small"""    )
							__lowerCAmelCase      		=       tokenizer("""Hello! I am the Funnel Transformer model.""",return_tensors="""pt"""    )
							__lowerCAmelCase      		=       model(**__SCREAMING_SNAKE_CASE    )[0]

							__lowerCAmelCase      		=       torch.tensor(235.7246    )
							__lowerCAmelCase      		=       torch.tensor(0.0256    )
							self.assertTrue(torch.allclose(output.sum(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )
							self.assertTrue(torch.allclose(output.mean(),__SCREAMING_SNAKE_CASE,atol=1e-4    )    )








'''simple docstring'''







import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


_a      :				Tuple											=					datasets.utils.logging.get_logger(__name__)





@dataclass
class 					_UpperCAmelCase							(		datasets.BuilderConfig     ):
		a     :     Optional[datasets.Features]     						=None
		a     :     str     						="utf-8"
		a     :     Optional[str]     						=None
		a     :     int     						=10 << 20  # 10MB
		a     :     bool     						=False
		a     :     str     						="line"





class 					_UpperCAmelCase							(		datasets.ArrowBasedBuilder     ):
		a     :     Optional[int]     						=TextConfig
		def 						lowerCamelCase__    (						self    ):



					'''simple docstring'''

					return datasets.DatasetInfo(features=self.config.features    )
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					if not self.config.data_files:
								raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'    )
					__lowerCAmelCase      		=       dl_manager.download_and_extract(self.config.data_files    )
					if isinstance(__SCREAMING_SNAKE_CASE,(str, list, tuple)    ):
								__lowerCAmelCase      		=       data_files
								if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ):
											__lowerCAmelCase      		=       [files]
								__lowerCAmelCase      		=       [dl_manager.iter_files(__SCREAMING_SNAKE_CASE    ) for file in files]
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN,gen_kwargs={"""files""": files}    )]
					__lowerCAmelCase      		=       []
					for split_name, files in data_files.items():
								if isinstance(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    ):
											__lowerCAmelCase      		=       [files]
								__lowerCAmelCase      		=       [dl_manager.iter_files(__SCREAMING_SNAKE_CASE    ) for file in files]
								splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE,gen_kwargs={"""files""": files}    )    )
					return splits
		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					if self.config.features is not None:
								__lowerCAmelCase      		=       self.config.features.arrow_schema
								if all(not require_storage_cast(__SCREAMING_SNAKE_CASE    ) for feature in self.config.features.values()    ):
											# cheaper cast
											__lowerCAmelCase      		=       pa_table.cast(__SCREAMING_SNAKE_CASE    )
								else:
											# more expensive cast; allows str <-> int/float or str to Audio for example
											__lowerCAmelCase      		=       table_cast(__SCREAMING_SNAKE_CASE,__SCREAMING_SNAKE_CASE    )
								return pa_table
					else:
								return pa_table.cast(pa.schema({"""text""": pa.string()}    )    )







		def 						lowerCamelCase__    (						self,__SCREAMING_SNAKE_CASE    ):



					'''simple docstring'''

					__lowerCAmelCase      		=       list(self.config.features    ) if self.config.features is not None else ["""text"""]
					for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE    )    ):
								# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
								with open(__SCREAMING_SNAKE_CASE,encoding=self.config.encoding,errors=self.config.errors    ) as f:
											if self.config.sample_by == "line":
														__lowerCAmelCase      		=       0
														while True:
																	__lowerCAmelCase      		=       f.read(self.config.chunksize    )
																	if not batch:
																				break
																	batch += f.readline()  # finish current line
																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																	__lowerCAmelCase      		=       StringIO(__SCREAMING_SNAKE_CASE    ).readlines()
																	if not self.config.keep_linebreaks:
																				__lowerCAmelCase      		=       [line.rstrip("""\n"""    ) for line in batch]
																	__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array(__SCREAMING_SNAKE_CASE    )],names=__SCREAMING_SNAKE_CASE    )
																	# Uncomment for debugging (will print the Arrow table size and elements)
																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
																	batch_idx += 1
											elif self.config.sample_by == "paragraph":
														__lowerCAmelCase      		=       0
														__lowerCAmelCase      		=       """"""
														while True:
																	__lowerCAmelCase      		=       f.read(self.config.chunksize    )
																	if not new_batch:
																				break
																	batch += new_batch
																	batch += f.readline()  # finish current line
																	__lowerCAmelCase      		=       batch.split("""\n\n"""    )
																	__lowerCAmelCase      		=       pa.Table.from_arrays(
																	    [pa.array([example for example in batch[:-1] if example]    )],names=__SCREAMING_SNAKE_CASE    )
																	# Uncomment for debugging (will print the Arrow table size and elements)
																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
																	batch_idx += 1
																	__lowerCAmelCase      		=       batch[-1]
														if batch:
																	__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array([batch]    )],names=__SCREAMING_SNAKE_CASE    )
																	yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE    )
											elif self.config.sample_by == "document":
														__lowerCAmelCase      		=       f.read()
														__lowerCAmelCase      		=       pa.Table.from_arrays([pa.array([text]    )],names=__SCREAMING_SNAKE_CASE    )
														yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE    )

