


'''simple docstring'''

import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           MODEL_FOR_PRETRAINING_MAPPING,
           FunnelBaseModel,
           FunnelForMaskedLM,
           FunnelForMultipleChoice,
           FunnelForPreTraining,
           FunnelForQuestionAnswering,
           FunnelForSequenceClassification,
           FunnelForTokenClassification,
           FunnelModel,
       )







class  UpperCAmelCase :

 def __init__( self :	str,					a_ :	Optional[int],					a_ :	Optional[int]=13,					a_ :	List[str]=7,					a_ :	int=True,					a_ :	Optional[Any]=True,					a_ :	Optional[int]=True,					a_ :	Union[str, Any]=True,					a_ :	Union[str, Any]=99,					a_ :	str=[1, 1, 2],					a_ :	Optional[Any]=1,					a_ :	int=32,					a_ :	List[Any]=4,					a_ :	Optional[int]=8,					a_ :	int=37,					a_ :	Any="gelu_new",					a_ :	Optional[Any]=0.1,					a_ :	str=0.1,					a_ :	Tuple=0.0,					a_ :	Optional[Any]=512,					a_ :	str=3,					a_ :	Union[str, Any]=0.02,					a_ :	Union[str, Any]=3,					a_ :	str=4,					a_ :	Dict=None,					a_ :	Dict=False,					):





    """simple docstring"""



    UpperCamelCase__ 						=						parent
    UpperCamelCase__ 						=						batch_size
    UpperCamelCase__ 						=						seq_length
    UpperCamelCase__ 						=						is_training
    UpperCamelCase__ 						=						use_input_mask
    UpperCamelCase__ 						=						use_token_type_ids
    UpperCamelCase__ 						=						use_labels
    UpperCamelCase__ 						=						vocab_size
    UpperCamelCase__ 						=						block_sizes
    UpperCamelCase__ 						=						num_decoder_layers
    UpperCamelCase__ 						=						d_model
    UpperCamelCase__ 						=						n_head
    UpperCamelCase__ 						=						d_head
    UpperCamelCase__ 						=						d_inner
    UpperCamelCase__ 						=						hidden_act
    UpperCamelCase__ 						=						hidden_dropout
    UpperCamelCase__ 						=						attention_dropout
    UpperCamelCase__ 						=						activation_dropout
    UpperCamelCase__ 						=						max_position_embeddings
    UpperCamelCase__ 						=						type_vocab_size
    UpperCamelCase__ 						=						2
    UpperCamelCase__ 						=						num_labels
    UpperCamelCase__ 						=						num_choices
    UpperCamelCase__ 						=						scope
    UpperCamelCase__ 						=						initializer_std

    # Used in the tests to check the size of the first attention layer
    UpperCamelCase__ 						=						n_head
    # Used in the tests to check the size of the first hidden state
    UpperCamelCase__ 						=						self.d_model
    # Used in the tests to check the number of output hidden states/attentions
    UpperCamelCase__ 						=						sum(self.block_sizes					) + (0 if base else self.num_decoder_layers)
    # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
    # the last hidden state of the first block (which is the first hidden state of the decoder).
    if not base:
       UpperCamelCase__ 						=						self.num_hidden_layers + 2

 def 		lowercase_						( self :	Dict					):





    """simple docstring"""



    UpperCamelCase__ 						=						ids_tensor([self.batch_size, self.seq_length],					self.vocab_size					)

    UpperCamelCase__ 						=						None
    if self.use_input_mask:
       UpperCamelCase__ 						=						ids_tensor([self.batch_size, self.seq_length],					vocab_size=2					)

    UpperCamelCase__ 						=						None
    if self.use_token_type_ids:
       UpperCamelCase__ 						=						ids_tensor([self.batch_size, self.seq_length],					self.type_vocab_size					)

    UpperCamelCase__ 						=						None
    UpperCamelCase__ 						=						None
    UpperCamelCase__ 						=						None
    if self.use_labels:
       UpperCamelCase__ 						=						ids_tensor([self.batch_size],					self.type_sequence_label_size					)
       UpperCamelCase__ 						=						ids_tensor([self.batch_size, self.seq_length],					self.num_labels					)
       UpperCamelCase__ 						=						ids_tensor([self.batch_size],					self.num_choices					)
       UpperCamelCase__ 						=						ids_tensor([self.batch_size, self.seq_length],					1					)

    UpperCamelCase__ 						=						self.get_config()

    return (
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
        fake_token_labels,
    )

 def 		lowercase_						( self :	str					):





    """simple docstring"""



    return FunnelConfig(
        vocab_size=self.vocab_size,					block_sizes=self.block_sizes,					num_decoder_layers=self.num_decoder_layers,					d_model=self.d_model,					n_head=self.n_head,					d_head=self.d_head,					d_inner=self.d_inner,					hidden_act=self.hidden_act,					hidden_dropout=self.hidden_dropout,					attention_dropout=self.attention_dropout,					activation_dropout=self.activation_dropout,					max_position_embeddings=self.max_position_embeddings,					type_vocab_size=self.type_vocab_size,					initializer_std=self.initializer_std,					)

 def 		lowercase_						( self :	str,					a_ :	int,					a_ :	List[str],					a_ :	Optional[Any],					a_ :	Tuple,					a_ :	List[Any],					a_ :	List[str],					a_ :	Tuple,					a_ :	Tuple,					):





    """simple docstring"""



    UpperCamelCase__ 						=						FunnelModel(config=a_					)
    model.to(a_					)
    model.eval()
    UpperCamelCase__ 						=						model(a_,					attention_mask=a_,					token_type_ids=a_					)
    UpperCamelCase__ 						=						model(a_,					token_type_ids=a_					)
    UpperCamelCase__ 						=						model(a_					)
    self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.d_model)					)

    UpperCamelCase__ 						=						False
    UpperCamelCase__ 						=						model(a_					)
    self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.d_model)					)

    UpperCamelCase__ 						=						False
    UpperCamelCase__ 						=						model(a_					)
    self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.d_model)					)

 def 		lowercase_						( self :	Optional[Any],					a_ :	Dict,					a_ :	int,					a_ :	Dict,					a_ :	Optional[int],					a_ :	Optional[Any],					a_ :	List[Any],					a_ :	Optional[Any],					a_ :	Union[str, Any],					):





    """simple docstring"""



    UpperCamelCase__ 						=						FunnelBaseModel(config=a_					)
    model.to(a_					)
    model.eval()
    UpperCamelCase__ 						=						model(a_,					attention_mask=a_,					token_type_ids=a_					)
    UpperCamelCase__ 						=						model(a_,					token_type_ids=a_					)
    UpperCamelCase__ 						=						model(a_					)
    self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, 2, self.d_model)					)

    UpperCamelCase__ 						=						False
    UpperCamelCase__ 						=						model(a_					)
    self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, 3, self.d_model)					)

    UpperCamelCase__ 						=						False
    UpperCamelCase__ 						=						model(a_					)
    self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, 2, self.d_model)					)

 def 		lowercase_						( self :	Optional[Any],					a_ :	str,					a_ :	List[Any],					a_ :	str,					a_ :	List[Any],					a_ :	List[Any],					a_ :	str,					a_ :	Optional[int],					a_ :	int,					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.num_labels
    UpperCamelCase__ 						=						FunnelForPreTraining(config=a_					)
    model.to(a_					)
    model.eval()
    UpperCamelCase__ 						=						model(a_,					attention_mask=a_,					token_type_ids=a_,					labels=a_					)
    self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length)					)

 def 		lowercase_						( self :	List[Any],					a_ :	Optional[Any],					a_ :	Union[str, Any],					a_ :	Union[str, Any],					a_ :	str,					a_ :	Union[str, Any],					a_ :	Union[str, Any],					a_ :	Tuple,					a_ :	List[str],					):





    """simple docstring"""



    UpperCamelCase__ 						=						FunnelForMaskedLM(config=a_					)
    model.to(a_					)
    model.eval()
    UpperCamelCase__ 						=						model(a_,					attention_mask=a_,					token_type_ids=a_,					labels=a_					)
    self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.vocab_size)					)

 def 		lowercase_						( self :	List[Any],					a_ :	List[str],					a_ :	Optional[int],					a_ :	Optional[int],					a_ :	List[str],					a_ :	str,					a_ :	Optional[int],					a_ :	List[str],					a_ :	Tuple,					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.num_labels
    UpperCamelCase__ 						=						FunnelForSequenceClassification(a_					)
    model.to(a_					)
    model.eval()
    UpperCamelCase__ 						=						model(a_,					attention_mask=a_,					token_type_ids=a_,					labels=a_					)
    self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.num_labels)					)

 def 		lowercase_						( self :	Optional[int],					a_ :	str,					a_ :	Optional[int],					a_ :	Optional[Any],					a_ :	int,					a_ :	int,					a_ :	Optional[int],					a_ :	Union[str, Any],					a_ :	Tuple,					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.num_choices
    UpperCamelCase__ 						=						FunnelForMultipleChoice(config=a_					)
    model.to(a_					)
    model.eval()
    UpperCamelCase__ 						=						input_ids.unsqueeze(1					).expand(-1,					self.num_choices,					-1					).contiguous()
    UpperCamelCase__ 						=						token_type_ids.unsqueeze(1					).expand(-1,					self.num_choices,					-1					).contiguous()
    UpperCamelCase__ 						=						input_mask.unsqueeze(1					).expand(-1,					self.num_choices,					-1					).contiguous()
    UpperCamelCase__ 						=						model(
        a_,					attention_mask=a_,					token_type_ids=a_,					labels=a_,					)
    self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.num_choices)					)

 def 		lowercase_						( self :	str,					a_ :	Optional[Any],					a_ :	Optional[Any],					a_ :	Tuple,					a_ :	Optional[int],					a_ :	Tuple,					a_ :	str,					a_ :	Optional[int],					a_ :	Dict,					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.num_labels
    UpperCamelCase__ 						=						FunnelForTokenClassification(config=a_					)
    model.to(a_					)
    model.eval()
    UpperCamelCase__ 						=						model(a_,					attention_mask=a_,					token_type_ids=a_,					labels=a_					)
    self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.num_labels)					)

 def 		lowercase_						( self :	Dict,					a_ :	str,					a_ :	Union[str, Any],					a_ :	int,					a_ :	int,					a_ :	Union[str, Any],					a_ :	Optional[int],					a_ :	Optional[int],					a_ :	Optional[Any],					):





    """simple docstring"""



    UpperCamelCase__ 						=						FunnelForQuestionAnswering(config=a_					)
    model.to(a_					)
    model.eval()
    UpperCamelCase__ 						=						model(
        a_,					attention_mask=a_,					token_type_ids=a_,					start_positions=a_,					end_positions=a_,					)
    self.parent.assertEqual(result.start_logits.shape,					(self.batch_size, self.seq_length)					)
    self.parent.assertEqual(result.end_logits.shape,					(self.batch_size, self.seq_length)					)

 def 		lowercase_						( self :	str					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.prepare_config_and_inputs()
    (
        (
        UpperCamelCase__
    )						,		(
        UpperCamelCase__
    )						,		(
        UpperCamelCase__
    )						,		(
        UpperCamelCase__
    )						,		(
        UpperCamelCase__
    )						,		(
        UpperCamelCase__
    )						,		(
        UpperCamelCase__
    )						,		(
        UpperCamelCase__
    )						,		
    ) 						=						config_and_inputs
    UpperCamelCase__ 						=						{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
    return config, inputs_dict







@require_torch
class  UpperCAmelCase (       SCREAMING_SNAKE_CASE__		,			SCREAMING_SNAKE_CASE__		,			unittest.TestCase):
 _lowerCamelCase :					List[str]    					=					False
 _lowerCamelCase :					Dict    					=					False
 _lowerCamelCase :					Any    					=					(
     (
         FunnelModel,
         FunnelForMaskedLM,
         FunnelForPreTraining,
         FunnelForQuestionAnswering,
         FunnelForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 _lowerCamelCase :					Tuple    					=					(
     {
         'feature-extraction': (FunnelBaseModel, FunnelModel),
         'fill-mask': FunnelForMaskedLM,
         'question-answering': FunnelForQuestionAnswering,
         'text-classification': FunnelForSequenceClassification,
         'token-classification': FunnelForTokenClassification,
         'zero-shot': FunnelForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )

 def 		lowercase_						( self :	str,					a_ :	str,					a_ :	Tuple,					a_ :	List[Any]=False					):





    """simple docstring"""



    UpperCamelCase__ 						=						super()._prepare_for_class(a_,					a_,					return_labels=a_					)

    if return_labels:
       if model_class in get_values(a_					):
          UpperCamelCase__ 						=						torch.zeros(
              (self.model_tester.batch_size, self.model_tester.seq_length),					dtype=torch.long,					device=a_					)
    return inputs_dict

 def 		lowercase_						( self :	str					):





    """simple docstring"""



    UpperCamelCase__ 						=						FunnelModelTester(self					)
    UpperCamelCase__ 						=						ConfigTester(self,					config_class=a_					)

 def 		lowercase_						( self :	List[str]					):





    """simple docstring"""



    self.config_tester.run_common_tests()

 def 		lowercase_						( self :	List[str]					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*a_					)

 def 		lowercase_						( self :	str					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_pretraining(*a_					)

 def 		lowercase_						( self :	Optional[Any]					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_masked_lm(*a_					)

 def 		lowercase_						( self :	Optional[int]					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_token_classification(*a_					)

 def 		lowercase_						( self :	Optional[Any]					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_question_answering(*a_					)

 def 		lowercase_						( self :	int,					a_ :	Dict					):





    """simple docstring"""



    if hasattr(a_,					"weight"					) and module.weight is not None:
       module.weight.data.fill_(3					)
    if hasattr(a_,					"bias"					) and module.bias is not None:
       module.bias.data.fill_(3					)

    for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(a_,					a_					) and getattr(a_,					a_					) is not None:
          UpperCamelCase__ 						=						getattr(a_,					a_					)
          weight.data.fill_(3					)







@require_torch
class  UpperCAmelCase (       SCREAMING_SNAKE_CASE__		,			unittest.TestCase):
 _lowerCamelCase :					Optional[int]    					=					False
 _lowerCamelCase :					Union[str, Any]    					=					False
 _lowerCamelCase :					List[str]    					=					(
     (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
 )

 def 		lowercase_						( self :	str					):





    """simple docstring"""



    UpperCamelCase__ 						=						FunnelModelTester(self,					base=a_					)
    UpperCamelCase__ 						=						ConfigTester(self,					config_class=a_					)

 def 		lowercase_						( self :	Dict					):





    """simple docstring"""



    self.config_tester.run_common_tests()

 def 		lowercase_						( self :	Dict					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_base_model(*a_					)

 def 		lowercase_						( self :	Any					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_sequence_classification(*a_					)

 def 		lowercase_						( self :	Optional[Any]					):





    """simple docstring"""



    UpperCamelCase__ 						=						self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_multiple_choice(*a_					)

 def 		lowercase_						( self :	Dict					):





    """simple docstring"""



    UpperCamelCase__						,		UpperCamelCase__ 						=						self.model_tester.prepare_config_and_inputs_for_common()
    UpperCamelCase__ 						=						True

    for model_class in self.all_model_classes:
       if model_class.__name__ == "FunnelBaseModel":
          continue
       UpperCamelCase__ 						=						model_class(a_					)
       model.to(a_					)
       model.train()
       UpperCamelCase__ 						=						self._prepare_for_class(a_,					a_,					return_labels=a_					)
       UpperCamelCase__ 						=						model(**a_					).loss
       loss.backward()

 def 		lowercase_						( self :	Optional[Any],					a_ :	Tuple					):





    """simple docstring"""



    if hasattr(a_,					"weight"					) and module.weight is not None:
       module.weight.data.fill_(3					)
    if hasattr(a_,					"bias"					) and module.bias is not None:
       module.bias.data.fill_(3					)

    for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
       if hasattr(a_,					a_					) and getattr(a_,					a_					) is not None:
          UpperCamelCase__ 						=						getattr(a_,					a_					)
          weight.data.fill_(3					)







@require_torch
@require_sentencepiece
@require_tokenizers
class  UpperCAmelCase (       unittest.TestCase):

 def 		lowercase_						( self :	Union[str, Any]					):





    """simple docstring"""



    UpperCamelCase__ 						=						13
    UpperCamelCase__ 						=						7
    UpperCamelCase__ 						=						torch.arange(0,					batch_size * sequence_length					).long().reshape(a_,					a_					)
    UpperCamelCase__ 						=						[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
    UpperCamelCase__ 						=						torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]					)

    UpperCamelCase__ 						=						FunnelModel.from_pretrained("sgugger/funnel-random-tiny"					)
    UpperCamelCase__ 						=						model(a_,					token_type_ids=a_					)[0].abs()

    UpperCamelCase__ 						=						torch.tensor(2_344.8_352					)
    UpperCamelCase__ 						=						torch.tensor(0.8_052					)
    self.assertTrue(torch.allclose(output.sum(),					a_,					atol=1e-4					)					)
    self.assertTrue(torch.allclose(output.mean(),					a_,					atol=1e-4					)					)

    UpperCamelCase__ 						=						torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]					)
    UpperCamelCase__ 						=						model(a_,					attention_mask=a_,					token_type_ids=a_					)[0].abs()

    UpperCamelCase__ 						=						torch.tensor(2_343.8_425					)
    UpperCamelCase__ 						=						torch.tensor(0.8_049					)
    self.assertTrue(torch.allclose(output.sum(),					a_,					atol=1e-4					)					)
    self.assertTrue(torch.allclose(output.mean(),					a_,					atol=1e-4					)					)

 @slow
 def 		lowercase_						( self :	int					):





    """simple docstring"""



    UpperCamelCase__ 						=						FunnelTokenizer.from_pretrained("huggingface/funnel-small"					)
    UpperCamelCase__ 						=						FunnelModel.from_pretrained("huggingface/funnel-small"					)
    UpperCamelCase__ 						=						tokenizer("Hello! I am the Funnel Transformer model.",					return_tensors="pt"					)
    UpperCamelCase__ 						=						model(**a_					)[0]

    UpperCamelCase__ 						=						torch.tensor(235.7_246					)
    UpperCamelCase__ 						=						torch.tensor(0.0_256					)
    self.assertTrue(torch.allclose(output.sum(),					a_,					atol=1e-4					)					)
    self.assertTrue(torch.allclose(output.mean(),					a_,					atol=1e-4					)					)



'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


__lowercase:		int							=     {
    "configuration_open_llama": ["OPEN_LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP", "OpenLlamaConfig"],
}

try:
       if not is_sentencepiece_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __lowercase:		str							=     ["LlamaTokenizer"]

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __lowercase:		List[Any]							=     ["LlamaTokenizerFast"]

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __lowercase:		Union[str, Any]							=     [
           "OpenLlamaForCausalLM",
           "OpenLlamaModel",
           "OpenLlamaPreTrainedModel",
           "OpenLlamaForSequenceClassification",
       ]


if TYPE_CHECKING:
       from .configuration_open_llama import OPEN_LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenLlamaConfig

       try:
              if not is_sentencepiece_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from transformers import LlamaTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from transformers import LlamaTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_open_llama import (
                  OpenLlamaForCausalLM,
                  OpenLlamaForSequenceClassification,
                  OpenLlamaModel,
                  OpenLlamaPreTrainedModel,
              )


else:
       import sys

       __lowercase:		Tuple							=     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)