


import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           FunnelBaseModel,
           FunnelForMaskedLM,
           FunnelForMultipleChoice,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForSequenceClassification,
           FunnelForTokenClassification,
           FunnelModel,
       )





class 			lowerCAmelCase__:






    '''simple docstring'''





    def __init__(						self					: Tuple							, __snake_case					: Any							, __snake_case					: Tuple=13							, __snake_case					: str=7							, __snake_case					: Optional[int]=True							, __snake_case					: Optional[Any]=True							, __snake_case					: Tuple=True							, __snake_case					: Any=True							, __snake_case					: Dict=99							, __snake_case					: Optional[int]=[1, 1, 2]							, __snake_case					: List[Any]=1							, __snake_case					: List[str]=32							, __snake_case					: str=4							, __snake_case					: Tuple=8							, __snake_case					: Dict=37							, __snake_case					: Any="gelu_new"							, __snake_case					: str=0.1							, __snake_case					: str=0.1							, __snake_case					: Optional[int]=0.0							, __snake_case					: str=512							, __snake_case					: List[Any]=3							, __snake_case					: Tuple=0.02							, __snake_case					: Union[str, Any]=3							, __snake_case					: Union[str, Any]=4							, __snake_case					: Any=None							, __snake_case					: List[str]=False							, ):





       '''simple docstring'''
       UpperCAmelCase_				:   Union[str, Any]							= parent
       UpperCAmelCase_				:   Union[str, Any]							= batch_size
       UpperCAmelCase_				:   Any							= seq_length
       UpperCAmelCase_				:   Union[str, Any]							= is_training
       UpperCAmelCase_				:   str							= use_input_mask
       UpperCAmelCase_				:   Any							= use_token_type_ids
       UpperCAmelCase_				:   List[Any]							= use_labels
       UpperCAmelCase_				:   Union[str, Any]							= vocab_size
       UpperCAmelCase_				:   int							= block_sizes
       UpperCAmelCase_				:   Optional[Any]							= num_decoder_layers
       UpperCAmelCase_				:   List[Any]							= d_model
       UpperCAmelCase_				:   List[Any]							= n_head
       UpperCAmelCase_				:   Dict							= d_head
       UpperCAmelCase_				:   Tuple							= d_inner
       UpperCAmelCase_				:   Dict							= hidden_act
       UpperCAmelCase_				:   Optional[int]							= hidden_dropout
       UpperCAmelCase_				:   Tuple							= attention_dropout
       UpperCAmelCase_				:   Optional[int]							= activation_dropout
       UpperCAmelCase_				:   List[Any]							= max_position_embeddings
       UpperCAmelCase_				:   Union[str, Any]							= type_vocab_size
       UpperCAmelCase_				:   int							= 2
       UpperCAmelCase_				:   int							= num_labels
       UpperCAmelCase_				:   Tuple							= num_choices
       UpperCAmelCase_				:   Optional[Any]							= scope
       UpperCAmelCase_				:   Union[str, Any]							= initializer_std

       # Used in the tests to check the size of the first attention layer
       UpperCAmelCase_				:   Dict							= n_head
       # Used in the tests to check the size of the first hidden state
       UpperCAmelCase_				:   List[Any]							= self.d_model
       # Used in the tests to check the number of output hidden states/attentions
       UpperCAmelCase_				:   Optional[int]							= sum(self.block_sizes						) + (0 if base else self.num_decoder_layers)
       # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
       # the last hidden state of the first block (which is the first hidden state of the decoder).
       if not base:
          UpperCAmelCase_				:   List[Any]							= self.num_hidden_layers + 2





    def        _lowerCamelCase			(						self					: Optional[int]						):





       '''simple docstring'''
       UpperCAmelCase_				:   Dict							= ids_tensor([self.batch_size, self.seq_length]							, self.vocab_size						)

       UpperCAmelCase_				:   Any							= None
       if self.use_input_mask:
          UpperCAmelCase_				:   Optional[Any]							= ids_tensor([self.batch_size, self.seq_length]							, vocab_size=2						)

       UpperCAmelCase_				:   Optional[Any]							= None
       if self.use_token_type_ids:
          UpperCAmelCase_				:   Any							= ids_tensor([self.batch_size, self.seq_length]							, self.type_vocab_size						)

       UpperCAmelCase_				:   Union[str, Any]							= None
       UpperCAmelCase_				:   Optional[Any]							= None
       UpperCAmelCase_				:   Optional[Any]							= None
       if self.use_labels:
          UpperCAmelCase_				:   Optional[int]							= ids_tensor([self.batch_size]							, self.type_sequence_label_size						)
          UpperCAmelCase_				:   Optional[Any]							= ids_tensor([self.batch_size, self.seq_length]							, self.num_labels						)
          UpperCAmelCase_				:   str							= ids_tensor([self.batch_size]							, self.num_choices						)
          UpperCAmelCase_				:   Optional[Any]							= ids_tensor([self.batch_size, self.seq_length]							, 1						)

       UpperCAmelCase_				:   List[str]							= self.get_config()

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
           fake_token_labels,
       )





    def        _lowerCamelCase			(						self					: Tuple						):





       '''simple docstring'''
       return FunnelConfig(
           vocab_size=self.vocab_size							, block_sizes=self.block_sizes							, num_decoder_layers=self.num_decoder_layers							, d_model=self.d_model							, n_head=self.n_head							, d_head=self.d_head							, d_inner=self.d_inner							, hidden_act=self.hidden_act							, hidden_dropout=self.hidden_dropout							, attention_dropout=self.attention_dropout							, activation_dropout=self.activation_dropout							, max_position_embeddings=self.max_position_embeddings							, type_vocab_size=self.type_vocab_size							, initializer_std=self.initializer_std							, )





    def        _lowerCamelCase			(						self					: Any							, __snake_case					: str							, __snake_case					: Optional[Any]							, __snake_case					: Tuple							, __snake_case					: Dict							, __snake_case					: List[str]							, __snake_case					: Tuple							, __snake_case					: List[Any]							, __snake_case					: Union[str, Any]							, ):





       '''simple docstring'''
       UpperCAmelCase_				:   Tuple							= FunnelModel(config=__snake_case						)
       model.to(__snake_case						)
       model.eval()
       UpperCAmelCase_				:   Dict							= model(__snake_case							, attention_mask=__snake_case							, token_type_ids=__snake_case						)
       UpperCAmelCase_				:   List[Any]							= model(__snake_case							, token_type_ids=__snake_case						)
       UpperCAmelCase_				:   Union[str, Any]							= model(__snake_case						)
       self.parent.assertEqual(result.last_hidden_state.shape							, (self.batch_size, self.seq_length, self.d_model)						)

       UpperCAmelCase_				:   int							= False
       UpperCAmelCase_				:   Tuple							= model(__snake_case						)
       self.parent.assertEqual(result.last_hidden_state.shape							, (self.batch_size, self.seq_length, self.d_model)						)

       UpperCAmelCase_				:   Optional[Any]							= False
       UpperCAmelCase_				:   int							= model(__snake_case						)
       self.parent.assertEqual(result.last_hidden_state.shape							, (self.batch_size, self.seq_length, self.d_model)						)





    def        _lowerCamelCase			(						self					: List[Any]							, __snake_case					: Optional[Any]							, __snake_case					: str							, __snake_case					: Tuple							, __snake_case					: Optional[Any]							, __snake_case					: Tuple							, __snake_case					: Any							, __snake_case					: Dict							, __snake_case					: Tuple							, ):





       '''simple docstring'''
       UpperCAmelCase_				:   int							= FunnelBaseModel(config=__snake_case						)
       model.to(__snake_case						)
       model.eval()
       UpperCAmelCase_				:   List[str]							= model(__snake_case							, attention_mask=__snake_case							, token_type_ids=__snake_case						)
       UpperCAmelCase_				:   Tuple							= model(__snake_case							, token_type_ids=__snake_case						)
       UpperCAmelCase_				:   str							= model(__snake_case						)
       self.parent.assertEqual(result.last_hidden_state.shape							, (self.batch_size, 2, self.d_model)						)

       UpperCAmelCase_				:   Tuple							= False
       UpperCAmelCase_				:   List[str]							= model(__snake_case						)
       self.parent.assertEqual(result.last_hidden_state.shape							, (self.batch_size, 3, self.d_model)						)

       UpperCAmelCase_				:   Optional[Any]							= False
       UpperCAmelCase_				:   str							= model(__snake_case						)
       self.parent.assertEqual(result.last_hidden_state.shape							, (self.batch_size, 2, self.d_model)						)





    def        _lowerCamelCase			(						self					: Dict							, __snake_case					: List[str]							, __snake_case					: Optional[int]							, __snake_case					: List[str]							, __snake_case					: Optional[Any]							, __snake_case					: Union[str, Any]							, __snake_case					: Union[str, Any]							, __snake_case					: Dict							, __snake_case					: Optional[Any]							, ):





       '''simple docstring'''
       UpperCAmelCase_				:   Any							= self.num_labels
       UpperCAmelCase_				:   Optional[int]							= FunnelForPreTraining(config=__snake_case						)
       model.to(__snake_case						)
       model.eval()
       UpperCAmelCase_				:   Optional[Any]							= model(__snake_case							, attention_mask=__snake_case							, token_type_ids=__snake_case							, labels=__snake_case						)
       self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.seq_length)						)





    def        _lowerCamelCase			(						self					: Dict							, __snake_case					: Tuple							, __snake_case					: List[str]							, __snake_case					: Any							, __snake_case					: Optional[int]							, __snake_case					: str							, __snake_case					: List[str]							, __snake_case					: List[str]							, __snake_case					: Tuple							, ):





       '''simple docstring'''
       UpperCAmelCase_				:   List[Any]							= FunnelForMaskedLM(config=__snake_case						)
       model.to(__snake_case						)
       model.eval()
       UpperCAmelCase_				:   Any							= model(__snake_case							, attention_mask=__snake_case							, token_type_ids=__snake_case							, labels=__snake_case						)
       self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.seq_length, self.vocab_size)						)





    def        _lowerCamelCase			(						self					: Any							, __snake_case					: List[str]							, __snake_case					: Any							, __snake_case					: Union[str, Any]							, __snake_case					: Any							, __snake_case					: Optional[Any]							, __snake_case					: List[str]							, __snake_case					: Any							, __snake_case					: Dict							, ):





       '''simple docstring'''
       UpperCAmelCase_				:   Optional[Any]							= self.num_labels
       UpperCAmelCase_				:   Tuple							= FunnelForSequenceClassification(__snake_case						)
       model.to(__snake_case						)
       model.eval()
       UpperCAmelCase_				:   int							= model(__snake_case							, attention_mask=__snake_case							, token_type_ids=__snake_case							, labels=__snake_case						)
       self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.num_labels)						)





    def        _lowerCamelCase			(						self					: Optional[int]							, __snake_case					: Tuple							, __snake_case					: Union[str, Any]							, __snake_case					: Any							, __snake_case					: Dict							, __snake_case					: Any							, __snake_case					: Union[str, Any]							, __snake_case					: Tuple							, __snake_case					: Dict							, ):





       '''simple docstring'''
       UpperCAmelCase_				:   Dict							= self.num_choices
       UpperCAmelCase_				:   Optional[Any]							= FunnelForMultipleChoice(config=__snake_case						)
       model.to(__snake_case						)
       model.eval()
       UpperCAmelCase_				:   List[Any]							= input_ids.unsqueeze(1						).expand(-1							, self.num_choices							, -1						).contiguous()
       UpperCAmelCase_				:   Tuple							= token_type_ids.unsqueeze(1						).expand(-1							, self.num_choices							, -1						).contiguous()
       UpperCAmelCase_				:   str							= input_mask.unsqueeze(1						).expand(-1							, self.num_choices							, -1						).contiguous()
       UpperCAmelCase_				:   Optional[int]							= model(
           __snake_case							, attention_mask=__snake_case							, token_type_ids=__snake_case							, labels=__snake_case							, )
       self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.num_choices)						)





    def        _lowerCamelCase			(						self					: Dict							, __snake_case					: int							, __snake_case					: Optional[Any]							, __snake_case					: Optional[int]							, __snake_case					: int							, __snake_case					: Any							, __snake_case					: Union[str, Any]							, __snake_case					: Tuple							, __snake_case					: Optional[Any]							, ):





       '''simple docstring'''
       UpperCAmelCase_				:   Optional[Any]							= self.num_labels
       UpperCAmelCase_				:   Tuple							= FunnelForTokenClassification(config=__snake_case						)
       model.to(__snake_case						)
       model.eval()
       UpperCAmelCase_				:   Optional[Any]							= model(__snake_case							, attention_mask=__snake_case							, token_type_ids=__snake_case							, labels=__snake_case						)
       self.parent.assertEqual(result.logits.shape							, (self.batch_size, self.seq_length, self.num_labels)						)





    def        _lowerCamelCase			(						self					: Any							, __snake_case					: int							, __snake_case					: Optional[int]							, __snake_case					: Optional[int]							, __snake_case					: Dict							, __snake_case					: Dict							, __snake_case					: str							, __snake_case					: Dict							, __snake_case					: int							, ):





       '''simple docstring'''
       UpperCAmelCase_				:   List[Any]							= FunnelForQuestionAnswering(config=__snake_case						)
       model.to(__snake_case						)
       model.eval()
       UpperCAmelCase_				:   Dict							= model(
           __snake_case							, attention_mask=__snake_case							, token_type_ids=__snake_case							, start_positions=__snake_case							, end_positions=__snake_case							, )
       self.parent.assertEqual(result.start_logits.shape							, (self.batch_size, self.seq_length)						)
       self.parent.assertEqual(result.end_logits.shape							, (self.batch_size, self.seq_length)						)





    def        _lowerCamelCase			(						self					: Optional[Any]						):





       '''simple docstring'''
       UpperCAmelCase_				:   Dict							= self.prepare_config_and_inputs()
       (
           (
           UpperCAmelCase_
       )		,   (
           UpperCAmelCase_
       )		,   (
           UpperCAmelCase_
       )		,   (
           UpperCAmelCase_
       )		,   (
           UpperCAmelCase_
       )		,   (
           UpperCAmelCase_
       )		,   (
           UpperCAmelCase_
       )		,   (
           UpperCAmelCase_
       )		,   
       )				:   int							= config_and_inputs
       UpperCAmelCase_				:   Any							= {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict





@require_torch
class 			lowerCAmelCase__(     snake_case__     ,     snake_case__     ,     unittest.TestCase    ):






    '''simple docstring'''





    A_	:  Dict								=			False
    A_	:  List[Any]								=			False
    A_	:  str								=			(
        (
            FunnelModel,
            FunnelForMaskedLM,
            FunnelForPreTraining,
            FunnelForQuestionAnswering,
            FunnelForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    A_	:  Union[str, Any]								=			(
        {
            'feature-extraction': (FunnelBaseModel, FunnelModel),
            'fill-mask': FunnelForMaskedLM,
            'question-answering': FunnelForQuestionAnswering,
            'text-classification': FunnelForSequenceClassification,
            'token-classification': FunnelForTokenClassification,
            'zero-shot': FunnelForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )





    def        _lowerCamelCase			(						self					: List[str]							, __snake_case					: List[Any]							, __snake_case					: Tuple							, __snake_case					: int=False						):





       '''simple docstring'''
       UpperCAmelCase_				:   List[str]							= super()._prepare_for_class(__snake_case							, __snake_case							, return_labels=__snake_case						)

       if return_labels:
          if model_class in get_values(__snake_case						):
             UpperCAmelCase_				:   Union[str, Any]							= torch.zeros(
                 (self.model_tester.batch_size, self.model_tester.seq_length)							, dtype=torch.long							, device=__snake_case						)
       return inputs_dict





    def        _lowerCamelCase			(						self					: int						):





       '''simple docstring'''
       UpperCAmelCase_				:   Union[str, Any]							= FunnelModelTester(self						)
       UpperCAmelCase_				:   Union[str, Any]							= ConfigTester(self							, config_class=__snake_case						)





    def        _lowerCamelCase			(						self					: Optional[Any]						):





       '''simple docstring'''
       self.config_tester.run_common_tests()





    def        _lowerCamelCase			(						self					: List[Any]						):





       '''simple docstring'''
       UpperCAmelCase_				:   Optional[int]							= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*__snake_case						)





    def        _lowerCamelCase			(						self					: Optional[int]						):





       '''simple docstring'''
       UpperCAmelCase_				:   List[str]							= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_pretraining(*__snake_case						)





    def        _lowerCamelCase			(						self					: Tuple						):





       '''simple docstring'''
       UpperCAmelCase_				:   List[str]							= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*__snake_case						)





    def        _lowerCamelCase			(						self					: Dict						):





       '''simple docstring'''
       UpperCAmelCase_				:   Optional[Any]							= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*__snake_case						)





    def        _lowerCamelCase			(						self					: Union[str, Any]						):





       '''simple docstring'''
       UpperCAmelCase_				:   int							= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*__snake_case						)





    def        _lowerCamelCase			(						self					: Any							, __snake_case					: Any						):





       '''simple docstring'''
       if hasattr(__snake_case							, '''weight'''						) and module.weight is not None:
          module.weight.data.fill_(3						)
       if hasattr(__snake_case							, '''bias'''						) and module.bias is not None:
          module.bias.data.fill_(3						)

       for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
          if hasattr(__snake_case							, __snake_case						) and getattr(__snake_case							, __snake_case						) is not None:
             UpperCAmelCase_				:   Tuple							= getattr(__snake_case							, __snake_case						)
             weight.data.fill_(3						)





@require_torch
class 			lowerCAmelCase__(     snake_case__     ,     unittest.TestCase    ):






    '''simple docstring'''





    A_	:  int								=			False
    A_	:  int								=			False
    A_	:  int								=			(
        (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
    )





    def        _lowerCamelCase			(						self					: List[str]						):





       '''simple docstring'''
       UpperCAmelCase_				:   Dict							= FunnelModelTester(self							, base=__snake_case						)
       UpperCAmelCase_				:   Tuple							= ConfigTester(self							, config_class=__snake_case						)





    def        _lowerCamelCase			(						self					: Optional[int]						):





       '''simple docstring'''
       self.config_tester.run_common_tests()





    def        _lowerCamelCase			(						self					: Any						):





       '''simple docstring'''
       UpperCAmelCase_				:   Union[str, Any]							= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_base_model(*__snake_case						)





    def        _lowerCamelCase			(						self					: Dict						):





       '''simple docstring'''
       UpperCAmelCase_				:   Tuple							= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*__snake_case						)





    def        _lowerCamelCase			(						self					: Union[str, Any]						):





       '''simple docstring'''
       UpperCAmelCase_				:   int							= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*__snake_case						)





    def        _lowerCamelCase			(						self					: List[str]						):





       '''simple docstring'''
       UpperCAmelCase_		,   UpperCAmelCase_				:   Union[str, Any]							= self.model_tester.prepare_config_and_inputs_for_common()
       UpperCAmelCase_				:   List[str]							= True

       for model_class in self.all_model_classes:
          if model_class.__name__ == "FunnelBaseModel":
             continue
          UpperCAmelCase_				:   Optional[int]							= model_class(__snake_case						)
          model.to(__snake_case						)
          model.train()
          UpperCAmelCase_				:   Optional[int]							= self._prepare_for_class(__snake_case							, __snake_case							, return_labels=__snake_case						)
          UpperCAmelCase_				:   Any							= model(**__snake_case						).loss
          loss.backward()





    def        _lowerCamelCase			(						self					: Tuple							, __snake_case					: Dict						):





       '''simple docstring'''
       if hasattr(__snake_case							, '''weight'''						) and module.weight is not None:
          module.weight.data.fill_(3						)
       if hasattr(__snake_case							, '''bias'''						) and module.bias is not None:
          module.bias.data.fill_(3						)

       for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
          if hasattr(__snake_case							, __snake_case						) and getattr(__snake_case							, __snake_case						) is not None:
             UpperCAmelCase_				:   Optional[Any]							= getattr(__snake_case							, __snake_case						)
             weight.data.fill_(3						)





@require_torch
@require_sentencepiece
@require_tokenizers
class 			lowerCAmelCase__(     unittest.TestCase    ):






    '''simple docstring'''





    def        _lowerCamelCase			(						self					: Dict						):





       '''simple docstring'''
       UpperCAmelCase_				:   Any							= 13
       UpperCAmelCase_				:   Optional[Any]							= 7
       UpperCAmelCase_				:   str							= torch.arange(0							, batch_size * sequence_length						).long().reshape(__snake_case							, __snake_case						)
       UpperCAmelCase_				:   Optional[Any]							= [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
       UpperCAmelCase_				:   Dict							= torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]						)

       UpperCAmelCase_				:   str							= FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''						)
       UpperCAmelCase_				:   Optional[Any]							= model(__snake_case							, token_type_ids=__snake_case						)[0].abs()

       UpperCAmelCase_				:   Optional[int]							= torch.tensor(2_344.8_352						)
       UpperCAmelCase_				:   List[str]							= torch.tensor(0.8_052						)
       self.assertTrue(torch.allclose(output.sum()							, __snake_case							, atol=1E-4						)						)
       self.assertTrue(torch.allclose(output.mean()							, __snake_case							, atol=1E-4						)						)

       UpperCAmelCase_				:   str							= torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]						)
       UpperCAmelCase_				:   Optional[int]							= model(__snake_case							, attention_mask=__snake_case							, token_type_ids=__snake_case						)[0].abs()

       UpperCAmelCase_				:   List[str]							= torch.tensor(2_343.8_425						)
       UpperCAmelCase_				:   Optional[Any]							= torch.tensor(0.8_049						)
       self.assertTrue(torch.allclose(output.sum()							, __snake_case							, atol=1E-4						)						)
       self.assertTrue(torch.allclose(output.mean()							, __snake_case							, atol=1E-4						)						)





    @slow
    def        _lowerCamelCase			(						self					: List[str]						):





       '''simple docstring'''
       UpperCAmelCase_				:   str							= FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''						)
       UpperCAmelCase_				:   Any							= FunnelModel.from_pretrained('''huggingface/funnel-small'''						)
       UpperCAmelCase_				:   int							= tokenizer('''Hello! I am the Funnel Transformer model.'''							, return_tensors='''pt'''						)
       UpperCAmelCase_				:   Optional[Any]							= model(**__snake_case						)[0]

       UpperCAmelCase_				:   Tuple							= torch.tensor(235.7_246						)
       UpperCAmelCase_				:   List[str]							= torch.tensor(0.0_256						)
       self.assertTrue(torch.allclose(output.sum()							, __snake_case							, atol=1E-4						)						)
       self.assertTrue(torch.allclose(output.mean()							, __snake_case							, atol=1E-4						)						)



import unittest

import numpy as np

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
       import tensorflow as tf

       from transformers.activations_tf import get_tf_activation





@require_tf
class 			lowerCAmelCase__(     unittest.TestCase    ):






    '''simple docstring'''





    def        _lowerCamelCase			(						self					: Optional[int]						):





       '''simple docstring'''
       UpperCAmelCase_				:   str							= tf.constant([-100, -1.0, -0.1, 0, 0.1, 1.0, 100.0]						)
       UpperCAmelCase_				:   List[Any]							= get_tf_activation('''gelu'''						)
       UpperCAmelCase_				:   Any							= get_tf_activation('''gelu_10'''						)

       UpperCAmelCase_				:   int							= gelu(__snake_case						)
       UpperCAmelCase_				:   Union[str, Any]							= geluaa(__snake_case						)

       UpperCAmelCase_				:   Union[str, Any]							= tf.where(y_gelu_aa < 10.0							, 1.0							, 0.0						)

       self.assertEqual(tf.math.reduce_max(__snake_case						).numpy().item()							, 10.0						)
       self.assertTrue(np.allclose(y_gelu * clipped_mask							, y_gelu_aa * clipped_mask						)						)





    def        _lowerCamelCase			(						self					: Dict						):





       '''simple docstring'''
       get_tf_activation('''gelu'''						)
       get_tf_activation('''gelu_10'''						)
       get_tf_activation('''gelu_fast'''						)
       get_tf_activation('''gelu_new'''						)
       get_tf_activation('''glu'''						)
       get_tf_activation('''mish'''						)
       get_tf_activation('''quick_gelu'''						)
       get_tf_activation('''relu'''						)
       get_tf_activation('''sigmoid'''						)
       get_tf_activation('''silu'''						)
       get_tf_activation('''swish'''						)
       get_tf_activation('''tanh'''						)
       with self.assertRaises(__snake_case						):
          get_tf_activation('''bogus'''						)
       with self.assertRaises(__snake_case						):
          get_tf_activation(__snake_case						)