



'''simple docstring'''

from typing import Dict, Optional, Tuple

import torch



def   __lowerCamelCase  (   lowerCAmelCase_						)    ->   torch.Tensor:
		_a  :   Tuple				 =							boundaries[1] - boundaries[0]
		_a  :   List[Any]				 =							boundaries + step / 2
		_a  :   Optional[int]				 =							torch.cat([bin_centers, (bin_centers[-1] + step).unsqueeze(-1						)]		,    dim=0						)
		return bin_centers



def   __lowerCamelCase  (   lowerCAmelCase_		,    lowerCAmelCase_		,    )    ->   Tuple[torch.Tensor, torch.Tensor]:
		_a  :   Dict				 =							_calculate_bin_centers(lowerCAmelCase_						)
		return (
		    torch.sum(aligned_distance_error_probs * bin_centers		,    dim=-1						),
		    bin_centers[-1],
		)



def   __lowerCamelCase  (   lowerCAmelCase_		,    lowerCAmelCase_ = 31		,    lowerCAmelCase_ = 64		,    **lowerCAmelCase_		,    )    ->   Dict[str, torch.Tensor]:
		_a  :   Union[str, Any]				 =							torch.linspace(0		,    lowerCAmelCase_		,    steps=(no_bins - 1)		,    device=logits.device						)

		_a  :   List[str]				 =							torch.nn.functional.softmax(lowerCAmelCase_		,    dim=-1						)
		_a ,      _a  :   List[Any]				 =							_calculate_expected_aligned_error(
		    alignment_confidence_breaks=lowerCAmelCase_		,    aligned_distance_error_probs=lowerCAmelCase_		,    )

		return {
		    "aligned_confidence_probs": aligned_confidence_probs,
		    "predicted_aligned_error": predicted_aligned_error,
		    "max_predicted_aligned_error": max_predicted_aligned_error,
		}



def   __lowerCamelCase  (   lowerCAmelCase_		,    lowerCAmelCase_ = None		,    lowerCAmelCase_ = 31		,    lowerCAmelCase_ = 64		,    lowerCAmelCase_ = 1E-8		,    **lowerCAmelCase_		,    )    ->   torch.Tensor:
		if residue_weights is None:
				_a  :   Optional[int]				 =							logits.new_ones(logits.shape[-2]						)

		_a  :   List[Any]				 =							torch.linspace(0		,    lowerCAmelCase_		,    steps=(no_bins - 1)		,    device=logits.device						)

		_a  :   Union[str, Any]				 =							_calculate_bin_centers(lowerCAmelCase_						)
		torch.sum(lowerCAmelCase_						)
		_a  :   str				 =							logits.shape[-2]
		_a  :   int				 =							max(lowerCAmelCase_		,    19						)

		_a  :   Optional[Any]				 =							1.24 * (clipped_n - 15) ** (1.0 / 3) - 1.8

		_a  :   Union[str, Any]				 =							torch.nn.functional.softmax(lowerCAmelCase_		,    dim=-1						)

		_a  :   List[Any]				 =							1.0 / (1 + (bin_centers**2) / (da**2))
		_a  :   Optional[int]				 =							torch.sum(probs * tm_per_bin		,    dim=-1						)

		_a  :   Tuple				 =							residue_weights / (eps + residue_weights.sum())
		_a  :   Union[str, Any]				 =							torch.sum(predicted_tm_term * normed_residue_mask		,    dim=-1						)

		_a  :   Tuple				 =							per_alignment * residue_weights

		_a  :   List[Any]				 =							(weighted == torch.max(lowerCAmelCase_						)).nonzero()[0]
		return per_alignment[tuple(lowerCAmelCase_						)]









'''simple docstring'''

import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    FunnelBaseModel,
							    FunnelForMaskedLM,
							    FunnelForMultipleChoice,
							    FunnelForPreTraining,
							    FunnelForQuestionAnswering,
							    FunnelForSequenceClassification,
							    FunnelForTokenClassification,
							    FunnelModel,
							)

class 			__magic_name__		:







							def __init__(							self      : Tuple			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : int=13			,_UpperCAmelCase      : List[str]=7			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : List[Any]=True			,_UpperCAmelCase      : Tuple=True			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : str=99			,_UpperCAmelCase      : Optional[Any]=[1, 1, 2]			,_UpperCAmelCase      : Optional[Any]=1			,_UpperCAmelCase      : Any=32			,_UpperCAmelCase      : Optional[Any]=4			,_UpperCAmelCase      : Optional[int]=8			,_UpperCAmelCase      : str=37			,_UpperCAmelCase      : Tuple="gelu_new"			,_UpperCAmelCase      : Optional[int]=0.1			,_UpperCAmelCase      : List[Any]=0.1			,_UpperCAmelCase      : Dict=0.0			,_UpperCAmelCase      : str=512			,_UpperCAmelCase      : Any=3			,_UpperCAmelCase      : int=0.02			,_UpperCAmelCase      : str=3			,_UpperCAmelCase      : Tuple=4			,_UpperCAmelCase      : Optional[Any]=None			,_UpperCAmelCase      : Optional[int]=False			,):
									_a  :   Optional[Any]				 =							parent
									_a  :   Optional[Any]				 =							batch_size
									_a  :   Union[str, Any]				 =							seq_length
									_a  :   int				 =							is_training
									_a  :   List[Any]				 =							use_input_mask
									_a  :   Dict				 =							use_token_type_ids
									_a  :   List[str]				 =							use_labels
									_a  :   int				 =							vocab_size
									_a  :   Union[str, Any]				 =							block_sizes
									_a  :   int				 =							num_decoder_layers
									_a  :   List[str]				 =							d_model
									_a  :   Optional[Any]				 =							n_head
									_a  :   int				 =							d_head
									_a  :   Union[str, Any]				 =							d_inner
									_a  :   Any				 =							hidden_act
									_a  :   int				 =							hidden_dropout
									_a  :   str				 =							attention_dropout
									_a  :   Any				 =							activation_dropout
									_a  :   List[Any]				 =							max_position_embeddings
									_a  :   Optional[int]				 =							type_vocab_size
									_a  :   Optional[int]				 =							2
									_a  :   Dict				 =							num_labels
									_a  :   List[str]				 =							num_choices
									_a  :   List[str]				 =							scope
									_a  :   Any				 =							initializer_std

									# Used in the tests to check the size of the first attention layer
									_a  :   List[Any]				 =							n_head
									# Used in the tests to check the size of the first hidden state
									_a  :   List[Any]				 =							self.d_model
									# Used in the tests to check the number of output hidden states/attentions
									_a  :   Tuple				 =							sum(self.block_sizes	) + (0 if base else self.num_decoder_layers)
									# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
									# the last hidden state of the first block (which is the first hidden state of the decoder).
									if not base:
											_a  :   Tuple				 =							self.num_hidden_layers + 2







							def 							__lowercase    (							self      : Any	):
									_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.vocab_size	)

									_a  :   Dict				 =							None
									if self.use_input_mask:
											_a  :   str				 =							ids_tensor([self.batch_size, self.seq_length]			,vocab_size=2	)

									_a  :   Any				 =							None
									if self.use_token_type_ids:
											_a  :   Optional[int]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.type_vocab_size	)

									_a  :   str				 =							None
									_a  :   Optional[int]				 =							None
									_a  :   str				 =							None
									if self.use_labels:
											_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size]			,self.type_sequence_label_size	)
											_a  :   List[Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.num_labels	)
											_a  :   List[str]				 =							ids_tensor([self.batch_size]			,self.num_choices	)
											_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,1	)

									_a  :   Dict				 =							self.get_config()

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_mask,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									    fake_token_labels,
									)







							def 							__lowercase    (							self      : Any	):
									return FunnelConfig(
									    vocab_size=self.vocab_size			,block_sizes=self.block_sizes			,num_decoder_layers=self.num_decoder_layers			,d_model=self.d_model			,n_head=self.n_head			,d_head=self.d_head			,d_inner=self.d_inner			,hidden_act=self.hidden_act			,hidden_dropout=self.hidden_dropout			,attention_dropout=self.attention_dropout			,activation_dropout=self.activation_dropout			,max_position_embeddings=self.max_position_embeddings			,type_vocab_size=self.type_vocab_size			,initializer_std=self.initializer_std			,)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : List[str]			,):
									_a  :   int				 =							FunnelModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   int				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   List[str]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   int				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)

									_a  :   Optional[Any]				 =							False
									_a  :   str				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)

									_a  :   Tuple				 =							False
									_a  :   Dict				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Any			,):
									_a  :   int				 =							FunnelBaseModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Optional[Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   int				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   Tuple				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 2, self.d_model)	)

									_a  :   Dict				 =							False
									_a  :   Dict				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 3, self.d_model)	)

									_a  :   Tuple				 =							False
									_a  :   List[Any]				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 2, self.d_model)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any			,):
									_a  :   Union[str, Any]				 =							self.num_labels
									_a  :   Any				 =							FunnelForPreTraining(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length)	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,):
									_a  :   Tuple				 =							FunnelForMaskedLM(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   List[str]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.vocab_size)	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[int]			,):
									_a  :   List[str]				 =							self.num_labels
									_a  :   Dict				 =							FunnelForSequenceClassification(_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_labels)	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : int			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[int]			,):
									_a  :   int				 =							self.num_choices
									_a  :   Dict				 =							FunnelForMultipleChoice(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Any				 =							input_ids.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   List[Any]				 =							token_type_ids.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   Tuple				 =							input_mask.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   Union[str, Any]				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase			,)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_choices)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Any			,):
									_a  :   Any				 =							self.num_labels
									_a  :   int				 =							FunnelForTokenClassification(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Union[str, Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.num_labels)	)







							def 							__lowercase    (							self      : int			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Optional[Any]			,):
									_a  :   List[str]				 =							FunnelForQuestionAnswering(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Union[str, Any]				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,start_positions=_UpperCAmelCase			,end_positions=_UpperCAmelCase			,)
									self.parent.assertEqual(result.start_logits.shape			,(self.batch_size, self.seq_length)	)
									self.parent.assertEqual(result.end_logits.shape			,(self.batch_size, self.seq_length)	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Tuple				 =							self.prepare_config_and_inputs()
									(
									    (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      
									)  :   List[str]				 =							config_and_inputs
									_a  :   int				 =							{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
									return config, inputs_dict

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					int            =      False
							lowerCAmelCase :					List[Any]            =      False
							lowerCAmelCase :					Union[str, Any]            =      (
							    (
							        FunnelModel,
							        FunnelForMaskedLM,
							        FunnelForPreTraining,
							        FunnelForQuestionAnswering,
							        FunnelForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCAmelCase :					Optional[Any]            =      (
							    {
							        'feature-extraction': (FunnelBaseModel, FunnelModel),
							        'fill-mask': FunnelForMaskedLM,
							        'question-answering': FunnelForQuestionAnswering,
							        'text-classification': FunnelForSequenceClassification,
							        'token-classification': FunnelForTokenClassification,
							        'zero-shot': FunnelForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Optional[int]=False	):
									_a  :   List[Any]				 =							super()._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase			,return_labels=_UpperCAmelCase	)

									if return_labels:
											if model_class in get_values(_UpperCAmelCase	):
													_a  :   Optional[int]				 =							torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.seq_length)			,dtype=torch.long			,device=_UpperCAmelCase	)
									return inputs_dict







							def 							__lowercase    (							self      : List[Any]	):
									_a  :   int				 =							FunnelModelTester(self	)
									_a  :   List[str]				 =							ConfigTester(self			,config_class=_UpperCAmelCase	)







							def 							__lowercase    (							self      : Tuple	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : Any	):
									_a  :   Any				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Union[str, Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_pretraining(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Tuple				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Optional[int]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Union[str, Any]	):
									if hasattr(_UpperCAmelCase			,'weight'	) and module.weight is not None:
											module.weight.data.fill_(3	)
									if hasattr(_UpperCAmelCase			,'bias'	) and module.bias is not None:
											module.bias.data.fill_(3	)

									for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
											if hasattr(_UpperCAmelCase			,_UpperCAmelCase	) and getattr(_UpperCAmelCase			,_UpperCAmelCase	) is not None:
													_a  :   str				 =							getattr(_UpperCAmelCase			,_UpperCAmelCase	)
													weight.data.fill_(3	)

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					Optional[int]            =      False
							lowerCAmelCase :					Optional[Any]            =      False
							lowerCAmelCase :					Optional[Any]            =      (
							    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
							)







							def 							__lowercase    (							self      : int	):
									_a  :   Dict				 =							FunnelModelTester(self			,base=_UpperCAmelCase	)
									_a  :   Union[str, Any]				 =							ConfigTester(self			,config_class=_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[int]	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Optional[Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_base_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Optional[Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[str]	):
									_a ,      _a  :   Optional[int]				 =							self.model_tester.prepare_config_and_inputs_for_common()
									_a  :   str				 =							True

									for model_class in self.all_model_classes:
											if model_class.__name__ == "FunnelBaseModel":
													continue
											_a  :   Optional[int]				 =							model_class(_UpperCAmelCase	)
											model.to(_UpperCAmelCase	)
											model.train()
											_a  :   Dict				 =							self._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase			,return_labels=_UpperCAmelCase	)
											_a  :   Dict				 =							model(**_UpperCAmelCase	).loss
											loss.backward()







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : List[Any]	):
									if hasattr(_UpperCAmelCase			,'weight'	) and module.weight is not None:
											module.weight.data.fill_(3	)
									if hasattr(_UpperCAmelCase			,'bias'	) and module.bias is not None:
											module.bias.data.fill_(3	)

									for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
											if hasattr(_UpperCAmelCase			,_UpperCAmelCase	) and getattr(_UpperCAmelCase			,_UpperCAmelCase	) is not None:
													_a  :   Tuple				 =							getattr(_UpperCAmelCase			,_UpperCAmelCase	)
													weight.data.fill_(3	)

@require_torch
@require_sentencepiece
@require_tokenizers
class 			__magic_name__		(					unittest.TestCase     ):







							def 							__lowercase    (							self      : Any	):
									_a  :   List[Any]				 =							13
									_a  :   Dict				 =							7
									_a  :   Union[str, Any]				 =							torch.arange(0			,batch_size * sequence_length	).long().reshape(_UpperCAmelCase			,_UpperCAmelCase	)
									_a  :   Tuple				 =							[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
									_a  :   List[Any]				 =							torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]	)

									_a  :   Tuple				 =							FunnelModel.from_pretrained('sgugger/funnel-random-tiny'	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)[0].abs()

									_a  :   Optional[int]				 =							torch.tensor(23_44.83_52	)
									_a  :   Union[str, Any]				 =							torch.tensor(0.80_52	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)

									_a  :   Tuple				 =							torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)[0].abs()

									_a  :   Union[str, Any]				 =							torch.tensor(23_43.84_25	)
									_a  :   Tuple				 =							torch.tensor(0.80_49	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)







							@slow
							def 							__lowercase    (							self      : str	):
									_a  :   Union[str, Any]				 =							FunnelTokenizer.from_pretrained('huggingface/funnel-small'	)
									_a  :   Tuple				 =							FunnelModel.from_pretrained('huggingface/funnel-small'	)
									_a  :   Union[str, Any]				 =							tokenizer('Hello! I am the Funnel Transformer model.'			,return_tensors='pt'	)
									_a  :   str				 =							model(**_UpperCAmelCase	)[0]

									_a  :   str				 =							torch.tensor(2_35.72_46	)
									_a  :   List[str]				 =							torch.tensor(0.02_56	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)




