
import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       MODEL_FOR_PRETRAINING_MAPPING,
       FunnelBaseModel,
       FunnelForMaskedLM,
       FunnelForMultipleChoice,
       FunnelForPreTraining,
       FunnelForQuestionAnswering,
       FunnelForSequenceClassification,
       FunnelForTokenClassification,
       FunnelModel,
   )






class UpperCAmelCase     :





   '''simple docstring'''


   def __init__(      self  :							Optional[int]    ,    lowerCAmelCase_  :							Tuple    ,    lowerCAmelCase_  :							str=1_3    ,    lowerCAmelCase_  :							str=7    ,    lowerCAmelCase_  :							str=True    ,    lowerCAmelCase_  :							Dict=True    ,    lowerCAmelCase_  :							Optional[int]=True    ,    lowerCAmelCase_  :							Optional[Any]=True    ,    lowerCAmelCase_  :							Dict=9_9    ,    lowerCAmelCase_  :							Union[str, Any]=[1, 1, 2]    ,    lowerCAmelCase_  :							Tuple=1    ,    lowerCAmelCase_  :							Union[str, Any]=3_2    ,    lowerCAmelCase_  :							List[str]=4    ,    lowerCAmelCase_  :							Optional[int]=8    ,    lowerCAmelCase_  :							Union[str, Any]=3_7    ,    lowerCAmelCase_  :							Optional[int]="gelu_new"    ,    lowerCAmelCase_  :							List[str]=0.1    ,    lowerCAmelCase_  :							Optional[int]=0.1    ,    lowerCAmelCase_  :							List[Any]=0.0    ,    lowerCAmelCase_  :							str=5_1_2    ,    lowerCAmelCase_  :							str=3    ,    lowerCAmelCase_  :							Union[str, Any]=0.02    ,    lowerCAmelCase_  :							Optional[Any]=3    ,    lowerCAmelCase_  :							Optional[int]=4    ,    lowerCAmelCase_  :							Dict=None    ,    lowerCAmelCase_  :							Dict=False    ,    ):

      """simple docstring"""
      _A:						Any	    = parent
      _A:						Optional[int]	    = batch_size
      _A:						List[Any]	    = seq_length
      _A:						List[Any]	    = is_training
      _A:						Optional[int]	    = use_input_mask
      _A:						int	    = use_token_type_ids
      _A:						Dict	    = use_labels
      _A:						Any	    = vocab_size
      _A:						List[Any]	    = block_sizes
      _A:						List[str]	    = num_decoder_layers
      _A:						Tuple	    = d_model
      _A:						Optional[Any]	    = n_head
      _A:						List[str]	    = d_head
      _A:						List[Any]	    = d_inner
      _A:						str	    = hidden_act
      _A:						Union[str, Any]	    = hidden_dropout
      _A:						Dict	    = attention_dropout
      _A:						List[str]	    = activation_dropout
      _A:						Optional[int]	    = max_position_embeddings
      _A:						str	    = type_vocab_size
      _A:						Optional[Any]	    = 2
      _A:						List[Any]	    = num_labels
      _A:						Any	    = num_choices
      _A:						Optional[Any]	    = scope
      _A:						List[str]	    = initializer_std

      # Used in the tests to check the size of the first attention layer
      _A:						Any	    = n_head
      # Used in the tests to check the size of the first hidden state
      _A:						Any	    = self.d_model
      # Used in the tests to check the number of output hidden states/attentions
      _A:						int	    = sum(self.block_sizes  ) + (0 if base else self.num_decoder_layers)
      # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
      # the last hidden state of the first block (which is the first hidden state of the decoder).
      if not base:
         _A:						Union[str, Any]	    = self.num_hidden_layers + 2


   def 				__magic_name__       (      self  :							Any  ):

      """simple docstring"""
      _A:						str	    = ids_tensor([self.batch_size, self.seq_length]    ,    self.vocab_size  )

      _A:						List[Any]	    = None
      if self.use_input_mask:
         _A:						List[str]	    = ids_tensor([self.batch_size, self.seq_length]    ,    vocab_size=2  )

      _A:						Optional[Any]	    = None
      if self.use_token_type_ids:
         _A:						Any	    = ids_tensor([self.batch_size, self.seq_length]    ,    self.type_vocab_size  )

      _A:						Optional[int]	    = None
      _A:						List[str]	    = None
      _A:						Optional[int]	    = None
      if self.use_labels:
         _A:						Optional[int]	    = ids_tensor([self.batch_size]    ,    self.type_sequence_label_size  )
         _A:						Tuple	    = ids_tensor([self.batch_size, self.seq_length]    ,    self.num_labels  )
         _A:						str	    = ids_tensor([self.batch_size]    ,    self.num_choices  )
         _A:						List[Any]	    = ids_tensor([self.batch_size, self.seq_length]    ,    1  )

      _A:						Any	    = self.get_config()

      return (
          config,
          input_ids,
          token_type_ids,
          input_mask,
          sequence_labels,
          token_labels,
          choice_labels,
          fake_token_labels,
      )


   def 				__magic_name__       (      self  :							Tuple  ):

      """simple docstring"""
      return FunnelConfig(
          vocab_size=self.vocab_size    ,    block_sizes=self.block_sizes    ,    num_decoder_layers=self.num_decoder_layers    ,    d_model=self.d_model    ,    n_head=self.n_head    ,    d_head=self.d_head    ,    d_inner=self.d_inner    ,    hidden_act=self.hidden_act    ,    hidden_dropout=self.hidden_dropout    ,    attention_dropout=self.attention_dropout    ,    activation_dropout=self.activation_dropout    ,    max_position_embeddings=self.max_position_embeddings    ,    type_vocab_size=self.type_vocab_size    ,    initializer_std=self.initializer_std    ,    )


   def 				__magic_name__       (      self  :							List[Any]    ,    lowerCAmelCase_  :							Union[str, Any]    ,    lowerCAmelCase_  :							List[Any]    ,    lowerCAmelCase_  :							Optional[int]    ,    lowerCAmelCase_  :							Dict    ,    lowerCAmelCase_  :							Union[str, Any]    ,    lowerCAmelCase_  :							List[Any]    ,    lowerCAmelCase_  :							Union[str, Any]    ,    lowerCAmelCase_  :							Any    ,    ):

      """simple docstring"""
      _A:						Any	    = FunnelModel(config=lowerCAmelCase_  )
      model.to(lowerCAmelCase_  )
      model.eval()
      _A:						Optional[int]	    = model(lowerCAmelCase_    ,    attention_mask=lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_  )
      _A:						str	    = model(lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_  )
      _A:						Dict	    = model(lowerCAmelCase_  )
      self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, self.seq_length, self.d_model)  )

      _A:						List[Any]	    = False
      _A:						Optional[int]	    = model(lowerCAmelCase_  )
      self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, self.seq_length, self.d_model)  )

      _A:						List[Any]	    = False
      _A:						List[Any]	    = model(lowerCAmelCase_  )
      self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, self.seq_length, self.d_model)  )


   def 				__magic_name__       (      self  :							str    ,    lowerCAmelCase_  :							Tuple    ,    lowerCAmelCase_  :							str    ,    lowerCAmelCase_  :							Optional[int]    ,    lowerCAmelCase_  :							Dict    ,    lowerCAmelCase_  :							int    ,    lowerCAmelCase_  :							List[str]    ,    lowerCAmelCase_  :							List[str]    ,    lowerCAmelCase_  :							Union[str, Any]    ,    ):

      """simple docstring"""
      _A:						List[str]	    = FunnelBaseModel(config=lowerCAmelCase_  )
      model.to(lowerCAmelCase_  )
      model.eval()
      _A:						Union[str, Any]	    = model(lowerCAmelCase_    ,    attention_mask=lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_  )
      _A:						Dict	    = model(lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_  )
      _A:						Union[str, Any]	    = model(lowerCAmelCase_  )
      self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, 2, self.d_model)  )

      _A:						Any	    = False
      _A:						str	    = model(lowerCAmelCase_  )
      self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, 3, self.d_model)  )

      _A:						List[Any]	    = False
      _A:						str	    = model(lowerCAmelCase_  )
      self.parent.assertEqual(result.last_hidden_state.shape    ,    (self.batch_size, 2, self.d_model)  )


   def 				__magic_name__       (      self  :							Dict    ,    lowerCAmelCase_  :							str    ,    lowerCAmelCase_  :							Tuple    ,    lowerCAmelCase_  :							str    ,    lowerCAmelCase_  :							str    ,    lowerCAmelCase_  :							Optional[Any]    ,    lowerCAmelCase_  :							str    ,    lowerCAmelCase_  :							Any    ,    lowerCAmelCase_  :							Optional[Any]    ,    ):

      """simple docstring"""
      _A:						List[Any]	    = self.num_labels
      _A:						Any	    = FunnelForPreTraining(config=lowerCAmelCase_  )
      model.to(lowerCAmelCase_  )
      model.eval()
      _A:						List[Any]	    = model(lowerCAmelCase_    ,    attention_mask=lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_    ,    labels=lowerCAmelCase_  )
      self.parent.assertEqual(result.logits.shape    ,    (self.batch_size, self.seq_length)  )


   def 				__magic_name__       (      self  :							Any    ,    lowerCAmelCase_  :							Tuple    ,    lowerCAmelCase_  :							Optional[Any]    ,    lowerCAmelCase_  :							Optional[Any]    ,    lowerCAmelCase_  :							int    ,    lowerCAmelCase_  :							Tuple    ,    lowerCAmelCase_  :							str    ,    lowerCAmelCase_  :							Optional[Any]    ,    lowerCAmelCase_  :							Dict    ,    ):

      """simple docstring"""
      _A:						List[Any]	    = FunnelForMaskedLM(config=lowerCAmelCase_  )
      model.to(lowerCAmelCase_  )
      model.eval()
      _A:						List[Any]	    = model(lowerCAmelCase_    ,    attention_mask=lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_    ,    labels=lowerCAmelCase_  )
      self.parent.assertEqual(result.logits.shape    ,    (self.batch_size, self.seq_length, self.vocab_size)  )


   def 				__magic_name__       (      self  :							str    ,    lowerCAmelCase_  :							Optional[Any]    ,    lowerCAmelCase_  :							List[str]    ,    lowerCAmelCase_  :							Union[str, Any]    ,    lowerCAmelCase_  :							List[Any]    ,    lowerCAmelCase_  :							int    ,    lowerCAmelCase_  :							List[Any]    ,    lowerCAmelCase_  :							List[Any]    ,    lowerCAmelCase_  :							Tuple    ,    ):

      """simple docstring"""
      _A:						str	    = self.num_labels
      _A:						Dict	    = FunnelForSequenceClassification(lowerCAmelCase_  )
      model.to(lowerCAmelCase_  )
      model.eval()
      _A:						Optional[Any]	    = model(lowerCAmelCase_    ,    attention_mask=lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_    ,    labels=lowerCAmelCase_  )
      self.parent.assertEqual(result.logits.shape    ,    (self.batch_size, self.num_labels)  )


   def 				__magic_name__       (      self  :							Dict    ,    lowerCAmelCase_  :							List[Any]    ,    lowerCAmelCase_  :							List[str]    ,    lowerCAmelCase_  :							int    ,    lowerCAmelCase_  :							Any    ,    lowerCAmelCase_  :							Optional[int]    ,    lowerCAmelCase_  :							Optional[int]    ,    lowerCAmelCase_  :							int    ,    lowerCAmelCase_  :							Dict    ,    ):

      """simple docstring"""
      _A:						List[Any]	    = self.num_choices
      _A:						List[Any]	    = FunnelForMultipleChoice(config=lowerCAmelCase_  )
      model.to(lowerCAmelCase_  )
      model.eval()
      _A:						str	    = input_ids.unsqueeze(1  ).expand(-1    ,    self.num_choices    ,    -1  ).contiguous()
      _A:						Any	    = token_type_ids.unsqueeze(1  ).expand(-1    ,    self.num_choices    ,    -1  ).contiguous()
      _A:						Optional[Any]	    = input_mask.unsqueeze(1  ).expand(-1    ,    self.num_choices    ,    -1  ).contiguous()
      _A:						int	    = model(
          lowerCAmelCase_    ,    attention_mask=lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_    ,    labels=lowerCAmelCase_    ,    )
      self.parent.assertEqual(result.logits.shape    ,    (self.batch_size, self.num_choices)  )


   def 				__magic_name__       (      self  :							Union[str, Any]    ,    lowerCAmelCase_  :							str    ,    lowerCAmelCase_  :							Union[str, Any]    ,    lowerCAmelCase_  :							str    ,    lowerCAmelCase_  :							Union[str, Any]    ,    lowerCAmelCase_  :							Dict    ,    lowerCAmelCase_  :							List[Any]    ,    lowerCAmelCase_  :							Optional[int]    ,    lowerCAmelCase_  :							Any    ,    ):

      """simple docstring"""
      _A:						List[str]	    = self.num_labels
      _A:						Any	    = FunnelForTokenClassification(config=lowerCAmelCase_  )
      model.to(lowerCAmelCase_  )
      model.eval()
      _A:						Tuple	    = model(lowerCAmelCase_    ,    attention_mask=lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_    ,    labels=lowerCAmelCase_  )
      self.parent.assertEqual(result.logits.shape    ,    (self.batch_size, self.seq_length, self.num_labels)  )


   def 				__magic_name__       (      self  :							str    ,    lowerCAmelCase_  :							Optional[int]    ,    lowerCAmelCase_  :							Dict    ,    lowerCAmelCase_  :							Dict    ,    lowerCAmelCase_  :							Dict    ,    lowerCAmelCase_  :							Optional[Any]    ,    lowerCAmelCase_  :							Optional[Any]    ,    lowerCAmelCase_  :							Union[str, Any]    ,    lowerCAmelCase_  :							int    ,    ):

      """simple docstring"""
      _A:						Optional[Any]	    = FunnelForQuestionAnswering(config=lowerCAmelCase_  )
      model.to(lowerCAmelCase_  )
      model.eval()
      _A:						Optional[int]	    = model(
          lowerCAmelCase_    ,    attention_mask=lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_    ,    start_positions=lowerCAmelCase_    ,    end_positions=lowerCAmelCase_    ,    )
      self.parent.assertEqual(result.start_logits.shape    ,    (self.batch_size, self.seq_length)  )
      self.parent.assertEqual(result.end_logits.shape    ,    (self.batch_size, self.seq_length)  )


   def 				__magic_name__       (      self  :							Tuple  ):

      """simple docstring"""
      _A:						Dict	    = self.prepare_config_and_inputs()
      (
          (
          _A
      )    ,						(
          _A
      )    ,						(
          _A
      )    ,						(
          _A
      )    ,						(
          _A
      )    ,						(
          _A
      )    ,						(
          _A
      )    ,						(
          _A
      )    ,						
      ):						List[Any]	    = config_and_inputs
      _A:						Any	    = {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
      return config, inputs_dict






@require_torch
class UpperCAmelCase     (		SCREAMING_SNAKE_CASE__							,							SCREAMING_SNAKE_CASE__							,							unittest.TestCase     ):





   '''simple docstring'''
   __UpperCamelCase	:					Union[str, Any]				       =					False
   __UpperCamelCase	:					Tuple				       =					False
   __UpperCamelCase	:					Dict				       =					(
       (
           FunnelModel,
           FunnelForMaskedLM,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForTokenClassification,
       )
       if is_torch_available()
       else ()
   )
   __UpperCamelCase	:					List[str]				       =					(
       {
           '''feature-extraction''': (FunnelBaseModel, FunnelModel),
           '''fill-mask''': FunnelForMaskedLM,
           '''question-answering''': FunnelForQuestionAnswering,
           '''text-classification''': FunnelForSequenceClassification,
           '''token-classification''': FunnelForTokenClassification,
           '''zero-shot''': FunnelForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )


   def 				__magic_name__       (      self  :							Tuple    ,    lowerCAmelCase_  :							List[Any]    ,    lowerCAmelCase_  :							Optional[Any]    ,    lowerCAmelCase_  :							Union[str, Any]=False  ):

      """simple docstring"""
      _A:						Dict	    = super()._prepare_for_class(lowerCAmelCase_    ,    lowerCAmelCase_    ,    return_labels=lowerCAmelCase_  )

      if return_labels:
         if model_class in get_values(lowerCAmelCase_  ):
            _A:						Union[str, Any]	    = torch.zeros(
                (self.model_tester.batch_size, self.model_tester.seq_length)    ,    dtype=torch.long    ,    device=lowerCAmelCase_  )
      return inputs_dict


   def 				__magic_name__       (      self  :							Any  ):

      """simple docstring"""
      _A:						int	    = FunnelModelTester(self  )
      _A:						Optional[Any]	    = ConfigTester(self    ,    config_class=lowerCAmelCase_  )


   def 				__magic_name__       (      self  :							List[Any]  ):

      """simple docstring"""
      self.config_tester.run_common_tests()


   def 				__magic_name__       (      self  :							Optional[int]  ):

      """simple docstring"""
      _A:						Union[str, Any]	    = self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(*lowerCAmelCase_  )


   def 				__magic_name__       (      self  :							Any  ):

      """simple docstring"""
      _A:						Optional[int]	    = self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase_  )


   def 				__magic_name__       (      self  :							Optional[int]  ):

      """simple docstring"""
      _A:						Dict	    = self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase_  )


   def 				__magic_name__       (      self  :							List[str]  ):

      """simple docstring"""
      _A:						List[str]	    = self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase_  )


   def 				__magic_name__       (      self  :							List[Any]  ):

      """simple docstring"""
      _A:						Dict	    = self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase_  )


   def 				__magic_name__       (      self  :							str    ,    lowerCAmelCase_  :							Optional[Any]  ):

      """simple docstring"""
      if hasattr(lowerCAmelCase_    ,    '''weight'''  ) and module.weight is not None:
         module.weight.data.fill_(3  )
      if hasattr(lowerCAmelCase_    ,    '''bias'''  ) and module.bias is not None:
         module.bias.data.fill_(3  )

      for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
         if hasattr(lowerCAmelCase_    ,    lowerCAmelCase_  ) and getattr(lowerCAmelCase_    ,    lowerCAmelCase_  ) is not None:
            _A:						List[str]	    = getattr(lowerCAmelCase_    ,    lowerCAmelCase_  )
            weight.data.fill_(3  )






@require_torch
class UpperCAmelCase     (		SCREAMING_SNAKE_CASE__							,							unittest.TestCase     ):





   '''simple docstring'''
   __UpperCamelCase	:					Dict				       =					False
   __UpperCamelCase	:					Optional[int]				       =					False
   __UpperCamelCase	:					Optional[Any]				       =					(
       (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
   )


   def 				__magic_name__       (      self  :							Any  ):

      """simple docstring"""
      _A:						int	    = FunnelModelTester(self    ,    base=lowerCAmelCase_  )
      _A:						Tuple	    = ConfigTester(self    ,    config_class=lowerCAmelCase_  )


   def 				__magic_name__       (      self  :							Optional[int]  ):

      """simple docstring"""
      self.config_tester.run_common_tests()


   def 				__magic_name__       (      self  :							Any  ):

      """simple docstring"""
      _A:						int	    = self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_base_model(*lowerCAmelCase_  )


   def 				__magic_name__       (      self  :							Dict  ):

      """simple docstring"""
      _A:						Dict	    = self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase_  )


   def 				__magic_name__       (      self  :							str  ):

      """simple docstring"""
      _A:						Union[str, Any]	    = self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase_  )


   def 				__magic_name__       (      self  :							str  ):

      """simple docstring"""
      _A    ,						_A:						str	    = self.model_tester.prepare_config_and_inputs_for_common()
      _A:						str	    = True

      for model_class in self.all_model_classes:
         if model_class.__name__ == "FunnelBaseModel":
            continue
         _A:						Any	    = model_class(lowerCAmelCase_  )
         model.to(lowerCAmelCase_  )
         model.train()
         _A:						List[str]	    = self._prepare_for_class(lowerCAmelCase_    ,    lowerCAmelCase_    ,    return_labels=lowerCAmelCase_  )
         _A:						Dict	    = model(**lowerCAmelCase_  ).loss
         loss.backward()


   def 				__magic_name__       (      self  :							Tuple    ,    lowerCAmelCase_  :							Tuple  ):

      """simple docstring"""
      if hasattr(lowerCAmelCase_    ,    '''weight'''  ) and module.weight is not None:
         module.weight.data.fill_(3  )
      if hasattr(lowerCAmelCase_    ,    '''bias'''  ) and module.bias is not None:
         module.bias.data.fill_(3  )

      for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
         if hasattr(lowerCAmelCase_    ,    lowerCAmelCase_  ) and getattr(lowerCAmelCase_    ,    lowerCAmelCase_  ) is not None:
            _A:						Any	    = getattr(lowerCAmelCase_    ,    lowerCAmelCase_  )
            weight.data.fill_(3  )






@require_torch
@require_sentencepiece
@require_tokenizers
class UpperCAmelCase     (		unittest.TestCase     ):





   '''simple docstring'''


   def 				__magic_name__       (      self  :							Tuple  ):

      """simple docstring"""
      _A:						str	    = 1_3
      _A:						Tuple	    = 7
      _A:						Union[str, Any]	    = torch.arange(0    ,    batch_size * sequence_length  ).long().reshape(lowerCAmelCase_    ,    lowerCAmelCase_  )
      _A:						Tuple	    = [0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
      _A:						Tuple	    = torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]  )

      _A:						Optional[int]	    = FunnelModel.from_pretrained('''sgugger/funnel-random-tiny'''  )
      _A:						Dict	    = model(lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_  )[0].abs()

      _A:						str	    = torch.tensor(2344.8352  )
      _A:						Optional[int]	    = torch.tensor(0.8052  )
      self.assertTrue(torch.allclose(output.sum()    ,    lowerCAmelCase_    ,    atol=1e-4  )  )
      self.assertTrue(torch.allclose(output.mean()    ,    lowerCAmelCase_    ,    atol=1e-4  )  )

      _A:						Optional[Any]	    = torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]  )
      _A:						Optional[int]	    = model(lowerCAmelCase_    ,    attention_mask=lowerCAmelCase_    ,    token_type_ids=lowerCAmelCase_  )[0].abs()

      _A:						Optional[Any]	    = torch.tensor(2343.8425  )
      _A:						Any	    = torch.tensor(0.8049  )
      self.assertTrue(torch.allclose(output.sum()    ,    lowerCAmelCase_    ,    atol=1e-4  )  )
      self.assertTrue(torch.allclose(output.mean()    ,    lowerCAmelCase_    ,    atol=1e-4  )  )


   @slow
   def 				__magic_name__       (      self  :							Union[str, Any]  ):

      """simple docstring"""
      _A:						Union[str, Any]	    = FunnelTokenizer.from_pretrained('''huggingface/funnel-small'''  )
      _A:						int	    = FunnelModel.from_pretrained('''huggingface/funnel-small'''  )
      _A:						Optional[int]	    = tokenizer('''Hello! I am the Funnel Transformer model.'''    ,    return_tensors='''pt'''  )
      _A:						Union[str, Any]	    = model(**lowerCAmelCase_  )[0]

      _A:						int	    = torch.tensor(235.7246  )
      _A:						Tuple	    = torch.tensor(0.0256  )
      self.assertTrue(torch.allclose(output.sum()    ,    lowerCAmelCase_    ,    atol=1e-4  )  )
      self.assertTrue(torch.allclose(output.mean()    ,    lowerCAmelCase_    ,    atol=1e-4  )  )





# This test is meant to be run in on an instance with TPUs like this:
#
#   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
#
# Replace 8 with the number of TPU cores you have.
#

import sys
from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.utils import logging


UpperCAmelCase__	:  Union[str, Any]      =   logging.get_logger(__name__)


if is_torch_available():
   import torch
   from torch import nn
   from torch.utils.data import Dataset

   from transformers import Trainer






   class UpperCAmelCase     (		SCREAMING_SNAKE_CASE__     ):





      '''simple docstring'''


      def __init__(      self  :							List[Any]    ,    lowerCAmelCase_  :							int = 1_0_1  ):

         """simple docstring"""
         _A:						str	    = length


      def __len__(      self  :							List[Any]  ):

         """simple docstring"""
         return self.length


      def __getitem__(      self  :							Any    ,    lowerCAmelCase_  :							Any  ):

         """simple docstring"""
         return i






   class UpperCAmelCase     :





      '''simple docstring'''


      def __call__(      self  :							Any    ,    lowerCAmelCase_  :							Union[str, Any]  ):

         """simple docstring"""
         return {"input_ids": torch.tensor(lowerCAmelCase_  ), "labels": torch.tensor(lowerCAmelCase_  )}






   class UpperCAmelCase     (		nn.Module     ):





      '''simple docstring'''


      def __init__(      self  :							str  ):

         """simple docstring"""
         super().__init__()
         # Add some (unused) params otherwise DDP will complain.
         _A:						Tuple	    = nn.Linear(1_2_0    ,    8_0  )


      def 				__magic_name__       (      self  :							List[Any]    ,    lowerCAmelCase_  :							List[str]    ,    lowerCAmelCase_  :							List[str]=None  ):

         """simple docstring"""
         if labels is not None:
            return torch.tensor(0.0    ,    device=input_ids.device  ), input_ids
         else:
            return input_ids






def 						lowerCamelCase__							(      )   ->     Optional[int]:
   _A:						Any	    = HfArgumentParser((TrainingArguments,)   )
   sys.argv += ["--output_dir", "./examples"]
   _A:						Any	    = parser.parse_args_into_dataclasses()[0]

   logger.warning(
       f"""Process rank: {training_args.local_rank}, device: {training_args.device}, """
       f"""tpu_num_cores: {training_args.tpu_num_cores}"""	,	)

   # Essentially, what we want to verify in the distributed case is
   # that we get all samples back, in the right order.
   # (this is crucial for prediction for instance)
   for dataset_length in [10_01, 2_56, 15]:
      _A:						Any	    = DummyDataset(a   )

      def compute_metrics(a   ) -> Dict:
         _A:						Dict	    = list(range(len(a   )   )   )
         _A:						List[str]	    = p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
         return {"success": success}

      _A:						Optional[Any]	    = Trainer(
          model=DummyModel()	,	args=a	,	data_collator=DummyDataCollator()	,	eval_dataset=a	,	compute_metrics=a	,	)
      _A:						List[str]	    = trainer.evaluate()
      logger.info(a   )
      if metrics["eval_success"] is not True:
         logger.error(a   )
         exit(1   )

      _A:						Optional[Any]	    = trainer.predict(a   )
      logger.info(p.metrics   )
      if p.metrics["test_success"] is not True:
         logger.error(p.metrics   )
         exit(1   )

      _A:						str	    = 2

      _A:						Optional[Any]	    = trainer.evaluate()
      logger.info(a   )
      if metrics["eval_success"] is not True:
         logger.error(a   )
         exit(1   )

      _A:						Optional[Any]	    = trainer.predict(a   )
      logger.info(p.metrics   )
      if p.metrics["test_success"] is not True:
         logger.error(p.metrics   )
         exit(1   )

      _A:						str	    = None

   logger.info('''🔥 All distributed tests successful'''   )






def 						lowerCamelCase__							(      a   )   ->     Optional[Any]:
   # For xla_spawn (TPUs)
   main()


if __name__ == "__main__":
   main()



