



import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MODEL_FOR_PRETRAINING_MAPPING,
      FunnelBaseModel,
      FunnelForMaskedLM,
      FunnelForMultipleChoice,
      FunnelForPreTraining,
      FunnelForQuestionAnswering,
      FunnelForSequenceClassification,
      FunnelForTokenClassification,
      FunnelModel,
  )







class    lowerCamelCase__     :





     def __init__(self	,  UpperCAmelCase	,  UpperCAmelCase=1_3	,  UpperCAmelCase=7	,  UpperCAmelCase=True	,  UpperCAmelCase=True	,  UpperCAmelCase=True	,  UpperCAmelCase=True	,  UpperCAmelCase=9_9	,  UpperCAmelCase=[1, 1, 2]	,  UpperCAmelCase=1	,  UpperCAmelCase=3_2	,  UpperCAmelCase=4	,  UpperCAmelCase=8	,  UpperCAmelCase=3_7	,  UpperCAmelCase="gelu_new"	,  UpperCAmelCase=0.1	,  UpperCAmelCase=0.1	,  UpperCAmelCase=0.0	,  UpperCAmelCase=5_1_2	,  UpperCAmelCase=3	,  UpperCAmelCase=0.02	,  UpperCAmelCase=3	,  UpperCAmelCase=4	,  UpperCAmelCase=None	,  UpperCAmelCase=False	,  )  ->     Any:
      _lowercase							   =parent
      _lowercase							   =batch_size
      _lowercase							   =seq_length
      _lowercase							   =is_training
      _lowercase							   =use_input_mask
      _lowercase							   =use_token_type_ids
      _lowercase							   =use_labels
      _lowercase							   =vocab_size
      _lowercase							   =block_sizes
      _lowercase							   =num_decoder_layers
      _lowercase							   =d_model
      _lowercase							   =n_head
      _lowercase							   =d_head
      _lowercase							   =d_inner
      _lowercase							   =hidden_act
      _lowercase							   =hidden_dropout
      _lowercase							   =attention_dropout
      _lowercase							   =activation_dropout
      _lowercase							   =max_position_embeddings
      _lowercase							   =type_vocab_size
      _lowercase							   =2
      _lowercase							   =num_labels
      _lowercase							   =num_choices
      _lowercase							   =scope
      _lowercase							   =initializer_std

      # Used in the tests to check the size of the first attention layer
      _lowercase							   =n_head
      # Used in the tests to check the size of the first hidden state
      _lowercase							   =self.d_model
      # Used in the tests to check the number of output hidden states/attentions
      _lowercase							   =sum(self.block_sizes ) + (0 if base else self.num_decoder_layers)
      # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
      # the last hidden state of the first block (which is the first hidden state of the decoder).
      if not base:
       _lowercase							   =self.num_hidden_layers + 2





     def   __A      (self )  ->     Any:
      _lowercase							   =ids_tensor([self.batch_size, self.seq_length]	,  self.vocab_size )

      _lowercase							   =None
      if self.use_input_mask:
       _lowercase							   =ids_tensor([self.batch_size, self.seq_length]	,  vocab_size=2 )

      _lowercase							   =None
      if self.use_token_type_ids:
       _lowercase							   =ids_tensor([self.batch_size, self.seq_length]	,  self.type_vocab_size )

      _lowercase							   =None
      _lowercase							   =None
      _lowercase							   =None
      if self.use_labels:
       _lowercase							   =ids_tensor([self.batch_size]	,  self.type_sequence_label_size )
       _lowercase							   =ids_tensor([self.batch_size, self.seq_length]	,  self.num_labels )
       _lowercase							   =ids_tensor([self.batch_size]	,  self.num_choices )
       _lowercase							   =ids_tensor([self.batch_size, self.seq_length]	,  1 )

      _lowercase							   =self.get_config()

      return (
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          fake_token_labels,
      )





     def   __A      (self )  ->     int:
      return FunnelConfig(
          vocab_size=self.vocab_size	,  block_sizes=self.block_sizes	,  num_decoder_layers=self.num_decoder_layers	,  d_model=self.d_model	,  n_head=self.n_head	,  d_head=self.d_head	,  d_inner=self.d_inner	,  hidden_act=self.hidden_act	,  hidden_dropout=self.hidden_dropout	,  attention_dropout=self.attention_dropout	,  activation_dropout=self.activation_dropout	,  max_position_embeddings=self.max_position_embeddings	,  type_vocab_size=self.type_vocab_size	,  initializer_std=self.initializer_std	,  )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  )  ->     Tuple:
      _lowercase							   =FunnelModel(config=__SCREAMING_SNAKE_CASE )
      model.to(__SCREAMING_SNAKE_CASE )
      model.eval()
      _lowercase							   =model(__SCREAMING_SNAKE_CASE	,  attention_mask=__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE )
      _lowercase							   =model(__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE )
      _lowercase							   =model(__SCREAMING_SNAKE_CASE )
      self.parent.assertEqual(result.last_hidden_state.shape	,  (self.batch_size, self.seq_length, self.d_model) )

      _lowercase							   =False
      _lowercase							   =model(__SCREAMING_SNAKE_CASE )
      self.parent.assertEqual(result.last_hidden_state.shape	,  (self.batch_size, self.seq_length, self.d_model) )

      _lowercase							   =False
      _lowercase							   =model(__SCREAMING_SNAKE_CASE )
      self.parent.assertEqual(result.last_hidden_state.shape	,  (self.batch_size, self.seq_length, self.d_model) )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  )  ->     Tuple:
      _lowercase							   =FunnelBaseModel(config=__SCREAMING_SNAKE_CASE )
      model.to(__SCREAMING_SNAKE_CASE )
      model.eval()
      _lowercase							   =model(__SCREAMING_SNAKE_CASE	,  attention_mask=__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE )
      _lowercase							   =model(__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE )
      _lowercase							   =model(__SCREAMING_SNAKE_CASE )
      self.parent.assertEqual(result.last_hidden_state.shape	,  (self.batch_size, 2, self.d_model) )

      _lowercase							   =False
      _lowercase							   =model(__SCREAMING_SNAKE_CASE )
      self.parent.assertEqual(result.last_hidden_state.shape	,  (self.batch_size, 3, self.d_model) )

      _lowercase							   =False
      _lowercase							   =model(__SCREAMING_SNAKE_CASE )
      self.parent.assertEqual(result.last_hidden_state.shape	,  (self.batch_size, 2, self.d_model) )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  )  ->     List[str]:
      _lowercase							   =self.num_labels
      _lowercase							   =FunnelForPreTraining(config=__SCREAMING_SNAKE_CASE )
      model.to(__SCREAMING_SNAKE_CASE )
      model.eval()
      _lowercase							   =model(__SCREAMING_SNAKE_CASE	,  attention_mask=__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE	,  labels=__SCREAMING_SNAKE_CASE )
      self.parent.assertEqual(result.logits.shape	,  (self.batch_size, self.seq_length) )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  )  ->     Optional[int]:
      _lowercase							   =FunnelForMaskedLM(config=__SCREAMING_SNAKE_CASE )
      model.to(__SCREAMING_SNAKE_CASE )
      model.eval()
      _lowercase							   =model(__SCREAMING_SNAKE_CASE	,  attention_mask=__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE	,  labels=__SCREAMING_SNAKE_CASE )
      self.parent.assertEqual(result.logits.shape	,  (self.batch_size, self.seq_length, self.vocab_size) )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  )  ->     str:
      _lowercase							   =self.num_labels
      _lowercase							   =FunnelForSequenceClassification(__SCREAMING_SNAKE_CASE )
      model.to(__SCREAMING_SNAKE_CASE )
      model.eval()
      _lowercase							   =model(__SCREAMING_SNAKE_CASE	,  attention_mask=__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE	,  labels=__SCREAMING_SNAKE_CASE )
      self.parent.assertEqual(result.logits.shape	,  (self.batch_size, self.num_labels) )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  )  ->     Any:
      _lowercase							   =self.num_choices
      _lowercase							   =FunnelForMultipleChoice(config=__SCREAMING_SNAKE_CASE )
      model.to(__SCREAMING_SNAKE_CASE )
      model.eval()
      _lowercase							   =input_ids.unsqueeze(1 ).expand(-1	,  self.num_choices	,  -1 ).contiguous()
      _lowercase							   =token_type_ids.unsqueeze(1 ).expand(-1	,  self.num_choices	,  -1 ).contiguous()
      _lowercase							   =input_mask.unsqueeze(1 ).expand(-1	,  self.num_choices	,  -1 ).contiguous()
      _lowercase							   =model(
          __SCREAMING_SNAKE_CASE	,  attention_mask=__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE	,  labels=__SCREAMING_SNAKE_CASE	,  )
      self.parent.assertEqual(result.logits.shape	,  (self.batch_size, self.num_choices) )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  )  ->     Optional[Any]:
      _lowercase							   =self.num_labels
      _lowercase							   =FunnelForTokenClassification(config=__SCREAMING_SNAKE_CASE )
      model.to(__SCREAMING_SNAKE_CASE )
      model.eval()
      _lowercase							   =model(__SCREAMING_SNAKE_CASE	,  attention_mask=__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE	,  labels=__SCREAMING_SNAKE_CASE )
      self.parent.assertEqual(result.logits.shape	,  (self.batch_size, self.seq_length, self.num_labels) )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase	,  )  ->     int:
      _lowercase							   =FunnelForQuestionAnswering(config=__SCREAMING_SNAKE_CASE )
      model.to(__SCREAMING_SNAKE_CASE )
      model.eval()
      _lowercase							   =model(
          __SCREAMING_SNAKE_CASE	,  attention_mask=__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE	,  start_positions=__SCREAMING_SNAKE_CASE	,  end_positions=__SCREAMING_SNAKE_CASE	,  )
      self.parent.assertEqual(result.start_logits.shape	,  (self.batch_size, self.seq_length) )
      self.parent.assertEqual(result.end_logits.shape	,  (self.batch_size, self.seq_length) )





     def   __A      (self )  ->     List[Any]:
      _lowercase							   =self.prepare_config_and_inputs()
      (
          (
          _lowercase 
      )   ,					(
          _lowercase 
      )   ,					(
          _lowercase 
      )   ,					(
          _lowercase 
      )   ,					(
          _lowercase 
      )   ,					(
          _lowercase 
      )   ,					(
          _lowercase 
      )   ,					(
          _lowercase 
      )   ,					 
      )							   =config_and_inputs
      _lowercase							   ={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
      return config, inputs_dict







@require_torch
class    lowerCamelCase__     ( lowerCAmelCase_     ,							lowerCAmelCase_     ,							unittest.TestCase):
     SCREAMING_SNAKE_CASE__       =    False
     SCREAMING_SNAKE_CASE__       =    False
     SCREAMING_SNAKE_CASE__       =    (
         (
             FunnelModel,
             FunnelForMaskedLM,
             FunnelForPreTraining,
             FunnelForQuestionAnswering,
             FunnelForTokenClassification,
         )
         if is_torch_available()
         else ()
     )
     SCREAMING_SNAKE_CASE__       =    (
         {
             """feature-extraction""": (FunnelBaseModel, FunnelModel),
             """fill-mask""": FunnelForMaskedLM,
             """question-answering""": FunnelForQuestionAnswering,
             """text-classification""": FunnelForSequenceClassification,
             """token-classification""": FunnelForTokenClassification,
             """zero-shot""": FunnelForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )





     def   __A      (self	,  UpperCAmelCase	,  UpperCAmelCase	,  UpperCAmelCase=False )  ->     Dict:
      _lowercase							   =super()._prepare_for_class(__SCREAMING_SNAKE_CASE	,  __SCREAMING_SNAKE_CASE	,  return_labels=__SCREAMING_SNAKE_CASE )

      if return_labels:
       if model_class in get_values(__SCREAMING_SNAKE_CASE ):
        _lowercase							   =torch.zeros(
            (self.model_tester.batch_size, self.model_tester.seq_length)	,  dtype=torch.long	,  device=__SCREAMING_SNAKE_CASE )
      return inputs_dict





     def   __A      (self )  ->     List[str]:
      _lowercase							   =FunnelModelTester(self )
      _lowercase							   =ConfigTester(self	,  config_class=__SCREAMING_SNAKE_CASE )





     def   __A      (self )  ->     int:
      self.config_tester.run_common_tests()





     def   __A      (self )  ->     Tuple:
      _lowercase							   =self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE )





     def   __A      (self )  ->     Optional[Any]:
      _lowercase							   =self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_pretraining(*__SCREAMING_SNAKE_CASE )





     def   __A      (self )  ->     str:
      _lowercase							   =self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_masked_lm(*__SCREAMING_SNAKE_CASE )





     def   __A      (self )  ->     Optional[int]:
      _lowercase							   =self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE )





     def   __A      (self )  ->     str:
      _lowercase							   =self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE )





     def   __A      (self	,  UpperCAmelCase )  ->     Union[str, Any]:

      if hasattr(__SCREAMING_SNAKE_CASE	,  '''weight''' ) and module.weight is not None:
       module.weight.data.fill_(3 )
      if hasattr(__SCREAMING_SNAKE_CASE	,  '''bias''' ) and module.bias is not None:
       module.bias.data.fill_(3 )

      for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(__SCREAMING_SNAKE_CASE	,  __SCREAMING_SNAKE_CASE ) and getattr(__SCREAMING_SNAKE_CASE	,  __SCREAMING_SNAKE_CASE ) is not None:
        _lowercase							   =getattr(__SCREAMING_SNAKE_CASE	,  __SCREAMING_SNAKE_CASE )
        weight.data.fill_(3 )







@require_torch
class    lowerCamelCase__     ( lowerCAmelCase_     ,							unittest.TestCase):
     SCREAMING_SNAKE_CASE__       =    False
     SCREAMING_SNAKE_CASE__       =    False
     SCREAMING_SNAKE_CASE__       =    (
         (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
     )





     def   __A      (self )  ->     Dict:
      _lowercase							   =FunnelModelTester(self	,  base=__SCREAMING_SNAKE_CASE )
      _lowercase							   =ConfigTester(self	,  config_class=__SCREAMING_SNAKE_CASE )





     def   __A      (self )  ->     Union[str, Any]:
      self.config_tester.run_common_tests()





     def   __A      (self )  ->     List[str]:
      _lowercase							   =self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_base_model(*__SCREAMING_SNAKE_CASE )





     def   __A      (self )  ->     Dict:
      _lowercase							   =self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE )





     def   __A      (self )  ->     List[str]:
      _lowercase							   =self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE )





     def   __A      (self )  ->     Optional[Any]:
      _lowercase   ,					_lowercase							   =self.model_tester.prepare_config_and_inputs_for_common()
      _lowercase							   =True

      for model_class in self.all_model_classes:
       if model_class.__name__ == "FunnelBaseModel":
        continue
       _lowercase							   =model_class(__SCREAMING_SNAKE_CASE )
       model.to(__SCREAMING_SNAKE_CASE )
       model.train()
       _lowercase							   =self._prepare_for_class(__SCREAMING_SNAKE_CASE	,  __SCREAMING_SNAKE_CASE	,  return_labels=__SCREAMING_SNAKE_CASE )
       _lowercase							   =model(**__SCREAMING_SNAKE_CASE ).loss
       loss.backward()





     def   __A      (self	,  UpperCAmelCase )  ->     Optional[Any]:

      if hasattr(__SCREAMING_SNAKE_CASE	,  '''weight''' ) and module.weight is not None:
       module.weight.data.fill_(3 )
      if hasattr(__SCREAMING_SNAKE_CASE	,  '''bias''' ) and module.bias is not None:
       module.bias.data.fill_(3 )

      for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(__SCREAMING_SNAKE_CASE	,  __SCREAMING_SNAKE_CASE ) and getattr(__SCREAMING_SNAKE_CASE	,  __SCREAMING_SNAKE_CASE ) is not None:
        _lowercase							   =getattr(__SCREAMING_SNAKE_CASE	,  __SCREAMING_SNAKE_CASE )
        weight.data.fill_(3 )







@require_torch
@require_sentencepiece
@require_tokenizers
class    lowerCamelCase__     ( unittest.TestCase):





     def   __A      (self )  ->     Optional[Any]:
      _lowercase							   =1_3
      _lowercase							   =7
      _lowercase							   =torch.arange(0	,  batch_size * sequence_length ).long().reshape(__SCREAMING_SNAKE_CASE	,  __SCREAMING_SNAKE_CASE )
      _lowercase							   =[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
      _lowercase							   =torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths] )

      _lowercase							   =FunnelModel.from_pretrained('''sgugger/funnel-random-tiny''' )
      _lowercase							   =model(__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE )[0].abs()

      _lowercase							   =torch.tensor(2344.8352 )
      _lowercase							   =torch.tensor(0.8052 )
      self.assertTrue(torch.allclose(output.sum()	,  __SCREAMING_SNAKE_CASE	,  atol=1e-4 ) )
      self.assertTrue(torch.allclose(output.mean()	,  __SCREAMING_SNAKE_CASE	,  atol=1e-4 ) )

      _lowercase							   =torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]] )
      _lowercase							   =model(__SCREAMING_SNAKE_CASE	,  attention_mask=__SCREAMING_SNAKE_CASE	,  token_type_ids=__SCREAMING_SNAKE_CASE )[0].abs()

      _lowercase							   =torch.tensor(2343.8425 )
      _lowercase							   =torch.tensor(0.8049 )
      self.assertTrue(torch.allclose(output.sum()	,  __SCREAMING_SNAKE_CASE	,  atol=1e-4 ) )
      self.assertTrue(torch.allclose(output.mean()	,  __SCREAMING_SNAKE_CASE	,  atol=1e-4 ) )





     @slow
     def   __A      (self )  ->     str:
      _lowercase							   =FunnelTokenizer.from_pretrained('''huggingface/funnel-small''' )
      _lowercase							   =FunnelModel.from_pretrained('''huggingface/funnel-small''' )
      _lowercase							   =tokenizer('''Hello! I am the Funnel Transformer model.'''	,  return_tensors='''pt''' )
      _lowercase							   =model(**__SCREAMING_SNAKE_CASE )[0]

      _lowercase							   =torch.tensor(235.7246 )
      _lowercase							   =torch.tensor(0.0256 )
      self.assertTrue(torch.allclose(output.sum()	,  __SCREAMING_SNAKE_CASE	,  atol=1e-4 ) )
      self.assertTrue(torch.allclose(output.mean()	,  __SCREAMING_SNAKE_CASE	,  atol=1e-4 ) )





import unittest

import numpy as np

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
  import tensorflow as tf

  from transformers.activations_tf import get_tf_activation







@require_tf
class    lowerCamelCase__     ( unittest.TestCase):





     def   __A      (self )  ->     Optional[Any]:
      _lowercase							   =tf.constant([-1_0_0, -1.0, -0.1, 0, 0.1, 1.0, 100.0] )
      _lowercase							   =get_tf_activation('''gelu''' )
      _lowercase							   =get_tf_activation('''gelu_10''' )

      _lowercase							   =gelu(UpperCAmelCase )
      _lowercase							   =geluaa(UpperCAmelCase )

      _lowercase							   =tf.where(y_gelu_aa < 10.0	,  1.0	,  0.0 )

      self.assertEqual(tf.math.reduce_max(UpperCAmelCase ).numpy().item()	,  10.0 )
      self.assertTrue(np.allclose(y_gelu * clipped_mask	,  y_gelu_aa * clipped_mask ) )





     def   __A      (self )  ->     Optional[Any]:
      get_tf_activation('''gelu''' )
      get_tf_activation('''gelu_10''' )
      get_tf_activation('''gelu_fast''' )
      get_tf_activation('''gelu_new''' )
      get_tf_activation('''glu''' )
      get_tf_activation('''mish''' )
      get_tf_activation('''quick_gelu''' )
      get_tf_activation('''relu''' )
      get_tf_activation('''sigmoid''' )
      get_tf_activation('''silu''' )
      get_tf_activation('''swish''' )
      get_tf_activation('''tanh''' )
      with self.assertRaises(UpperCAmelCase ):
       get_tf_activation('''bogus''' )
      with self.assertRaises(UpperCAmelCase ):
       get_tf_activation(UpperCAmelCase )

