



'''simple docstring'''

import unittest

import numpy as np
import timeout_decorator  # noqa

from transformers import OPTConfig, is_flax_available
from transformers.testing_utils import require_flax, require_sentencepiece, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor


if is_flax_available():
							import os

							# The slow tests are often failing with OOM error on GPU
							# This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed
							# but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html
							__lowerCAmelCase		    =     '''platform'''

							import jax
							import jax.numpy as jnp

							from transformers import FlaxOPTForCausalLM, FlaxOPTModel, GPTaTokenizer



def   __lowerCamelCase  (   lowerCAmelCase_		,    lowerCAmelCase_		,    lowerCAmelCase_=None		,    lowerCAmelCase_=None						)    ->   int:
		if attention_mask is None:
				_a  :   str				 =							np.where(input_ids != config.pad_token_id		,    1		,    0						)
		return {
		    "input_ids": input_ids,
		    "attention_mask": attention_mask,
		}

@require_flax
class 			__magic_name__		:







							def __init__(							self      : List[Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Optional[Any]=13			,_UpperCAmelCase      : Tuple=7			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : Dict=False			,_UpperCAmelCase      : List[Any]=99			,_UpperCAmelCase      : Optional[int]=16			,_UpperCAmelCase      : Any=2			,_UpperCAmelCase      : Tuple=4			,_UpperCAmelCase      : str=4			,_UpperCAmelCase      : Tuple="gelu"			,_UpperCAmelCase      : Union[str, Any]=0.1			,_UpperCAmelCase      : int=0.1			,_UpperCAmelCase      : Optional[Any]=20			,_UpperCAmelCase      : str=2			,_UpperCAmelCase      : Dict=1			,_UpperCAmelCase      : List[str]=0			,_UpperCAmelCase      : Optional[Any]=16			,_UpperCAmelCase      : List[str]=16			,_UpperCAmelCase      : Any=0.02			,):
									_a  :   int				 =							parent
									_a  :   List[str]				 =							batch_size
									_a  :   Tuple				 =							seq_length
									_a  :   List[str]				 =							is_training
									_a  :   Dict				 =							use_labels
									_a  :   str				 =							vocab_size
									_a  :   Tuple				 =							hidden_size
									_a  :   Tuple				 =							num_hidden_layers
									_a  :   List[str]				 =							num_attention_heads
									_a  :   int				 =							intermediate_size
									_a  :   str				 =							hidden_act
									_a  :   Tuple				 =							hidden_dropout_prob
									_a  :   Union[str, Any]				 =							attention_probs_dropout_prob
									_a  :   Dict				 =							max_position_embeddings
									_a  :   Tuple				 =							eos_token_id
									_a  :   Optional[Any]				 =							pad_token_id
									_a  :   Any				 =							bos_token_id
									_a  :   int				 =							embed_dim
									_a  :   int				 =							word_embed_proj_dim
									_a  :   Optional[int]				 =							initializer_range
									_a  :   Tuple				 =							False







							def 							__lowercase    (							self      : str	):
									_a  :   Tuple				 =							np.clip(ids_tensor([self.batch_size, self.seq_length - 1]			,self.vocab_size	)			,3			,self.vocab_size	)
									_a  :   List[str]				 =							np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1)			,dtype=np.intaa	))			,-1	)

									_a  :   Optional[int]				 =							OPTConfig(
									    vocab_size=self.vocab_size			,hidden_size=self.hidden_size			,num_hidden_layers=self.num_hidden_layers			,num_attention_heads=self.num_attention_heads			,ffn_dim=self.intermediate_size			,dropout=self.hidden_dropout_prob			,attention_dropout=self.attention_probs_dropout_prob			,max_position_embeddings=self.max_position_embeddings			,eos_token_id=self.eos_token_id			,bos_token_id=self.bos_token_id			,pad_token_id=self.pad_token_id			,embed_dim=self.embed_dim			,is_encoder_decoder=_UpperCAmelCase			,word_embed_proj_dim=self.word_embed_proj_dim			,initializer_range=self.initializer_range			,use_cache=_UpperCAmelCase			,)
									_a  :   List[Any]				 =							prepare_opt_inputs_dict(_UpperCAmelCase			,_UpperCAmelCase	)
									return config, inputs_dict







							def 							__lowercase    (							self      : Dict	):
									_a ,      _a  :   Union[str, Any]				 =							self.prepare_config_and_inputs()
									return config, inputs_dict







							def 							__lowercase    (							self      : List[str]			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : int	):
									_a  :   Dict				 =							20
									_a  :   int				 =							model_class_name(_UpperCAmelCase	)

									_a  :   Optional[int]				 =							inputs_dict['input_ids']
									_a  :   Tuple				 =							inputs_dict['attention_mask']

									_a  :   Optional[Any]				 =							model.init_cache(input_ids.shape[0]			,_UpperCAmelCase	)
									_a  :   List[Any]				 =							jnp.ones((input_ids.shape[0], max_length)			,dtype='i4'	)

									_a  :   Union[str, Any]				 =							jnp.broadcast_to(
									    jnp.arange(input_ids.shape[-1] - 1	)[None, :]			,(input_ids.shape[0], input_ids.shape[-1] - 1)			,)
									_a  :   Optional[int]				 =							model(
									    input_ids[:, :-1]			,attention_mask=_UpperCAmelCase			,past_key_values=_UpperCAmelCase			,position_ids=_UpperCAmelCase			,)

									_a  :   Tuple				 =							jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]			,dtype='i4'	)
									_a  :   Tuple				 =							model(
									    input_ids[:, -1:]			,attention_mask=_UpperCAmelCase			,past_key_values=outputs_cache.past_key_values			,position_ids=_UpperCAmelCase			,)

									_a  :   str				 =							model(_UpperCAmelCase	)

									_a  :   List[Any]				 =							np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])	)	)
									self.parent.assertTrue(diff < 1E-3			,msg=F"""Max diff is {diff}"""	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Any	):
									_a  :   Optional[Any]				 =							20
									_a  :   str				 =							model_class_name(_UpperCAmelCase	)

									_a ,      _a  :   Dict				 =							(
									    inputs_dict['input_ids'],
									    inputs_dict['attention_mask'],
									)

									_a  :   Tuple				 =							jnp.concatenate(
									    [
									        attention_mask,
									        jnp.zeros((attention_mask.shape[0], max_length - attention_mask.shape[1])	),
									    ]			,axis=-1			,)

									_a  :   Union[str, Any]				 =							model.init_cache(input_ids.shape[0]			,_UpperCAmelCase	)
									_a  :   Union[str, Any]				 =							jnp.broadcast_to(
									    jnp.arange(input_ids.shape[-1] - 1	)[None, :]			,(input_ids.shape[0], input_ids.shape[-1] - 1)			,)

									_a  :   List[str]				 =							model(
									    input_ids[:, :-1]			,attention_mask=_UpperCAmelCase			,past_key_values=_UpperCAmelCase			,position_ids=_UpperCAmelCase			,)
									_a  :   str				 =							jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]			,dtype='i4'	)
									_a  :   Dict				 =							model(
									    input_ids[:, -1:]			,past_key_values=outputs_cache.past_key_values			,attention_mask=_UpperCAmelCase			,position_ids=_UpperCAmelCase			,)

									_a  :   Optional[int]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase	)

									_a  :   int				 =							np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])	)	)
									self.parent.assertTrue(diff < 1E-3			,msg=F"""Max diff is {diff}"""	)

@require_flax
class 			__magic_name__		(					_UpperCamelCase    ,		unittest.TestCase    ,		_UpperCamelCase     ):
							lowerCAmelCase :					int            =      (FlaxOPTModel, FlaxOPTForCausalLM) if is_flax_available() else ()
							lowerCAmelCase :					Union[str, Any]            =      () if is_flax_available() else ()







							def 							__lowercase    (							self      : List[str]	):
									_a  :   Union[str, Any]				 =							FlaxOPTModelTester(self	)







							def 							__lowercase    (							self      : Any	):
									_a ,      _a  :   str				 =							self.model_tester.prepare_config_and_inputs()
									for model_class in self.all_model_classes:
											self.model_tester.check_use_cache_forward(_UpperCAmelCase			,_UpperCAmelCase			,_UpperCAmelCase	)







							def 							__lowercase    (							self      : Dict	):
									_a ,      _a  :   Tuple				 =							self.model_tester.prepare_config_and_inputs()
									for model_class in self.all_model_classes:
											self.model_tester.check_use_cache_forward_with_attn_mask(_UpperCAmelCase			,_UpperCAmelCase			,_UpperCAmelCase	)







							@slow
							def 							__lowercase    (							self      : Optional[int]	):
									for model_class_name in self.all_model_classes:
											_a  :   Optional[Any]				 =							model_class_name.from_pretrained('facebook/opt-125m'	)
											_a  :   Dict				 =							np.ones((1, 1)	) * model.config.eos_token_id
											_a  :   List[str]				 =							model(_UpperCAmelCase	)
											self.assertIsNotNone(_UpperCAmelCase	)

@require_sentencepiece
@require_flax
class 			__magic_name__		(					unittest.TestCase     ):







							@slow
							def 							__lowercase    (							self      : Any	):
									_a  :   List[str]				 =							FlaxOPTModel.from_pretrained('facebook/opt-350m'	)
									_a  :   Union[str, Any]				 =							jnp.array([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]	)
									_a  :   Optional[int]				 =							model(input_ids=_UpperCAmelCase	).last_hidden_state
									_a  :   str				 =							(1, 11, 512)
									self.assertEqual(output.shape			,_UpperCAmelCase	)
									_a  :   Any				 =							jnp.array(
									    [[-0.28_67, -1.92_56, -0.30_62], [-1.27_11, -0.13_37, -0.18_97], [0.41_09, 0.11_87, -1.31_42]]	)
									self.assertTrue(jnp.allclose(output[:, :3, :3]			,_UpperCAmelCase			,atol=4E-2	)	)

@require_flax
@slow
class 			__magic_name__		(					unittest.TestCase     ):







							def 							__lowercase    (							self      : Union[str, Any]	):
									super().setUp()
									_a  :   int				 =							'facebook/opt-350m'







							def 							__lowercase    (							self      : Dict	):
									_a  :   List[Any]				 =							FlaxOPTForCausalLM.from_pretrained(self.path_model	)
									_a  :   Dict				 =							GPTaTokenizer.from_pretrained(self.path_model	)

									_a  :   Tuple				 =							[
									    'Today is a beautiful day and I want to',
									    'In the city of',
									    'Paris is the capital of France and',
									    'Computers and mobile phones have taken',
									]
									# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
									_a  :   Tuple				 =							tokenizer(_UpperCAmelCase			,return_tensors='jax'			,padding=_UpperCAmelCase			,add_special_tokens=_UpperCAmelCase	)
									_a  :   int				 =							model(inputs.input_ids			,attention_mask=inputs.attention_mask	)[0].mean(axis=-1	)
									_a  :   Any				 =							jnp.array(
									    [
									        [1.38_51, -13.89_23, -10.52_29, -10.75_33, -0.23_09, -10.23_84, -0.53_65, -9.09_47, -5.16_70],
									        [-4.70_73, -10.62_76, -3.94_15, -21.52_42, -0.28_22, -0.28_22, -0.28_22, -0.28_22, -0.28_22],
									        [0.62_47, -3.42_29, -8.91_79, -1.42_97, -14.16_50, 1.41_46, -9.02_18, -0.27_03, -0.27_03],
									        [6.47_83, -1.99_13, -10.79_26, -2.33_36, 1.50_92, -0.99_74, -6.82_13, 1.34_77, 1.34_77],
									    ]	)
									self.assertTrue(jnp.allclose(_UpperCAmelCase			,_UpperCAmelCase			,atol=4E-2	)	)

									_a  :   List[Any]				 =							jax.jit(_UpperCAmelCase	)
									_a  :   Optional[Any]				 =							model(inputs.input_ids			,attention_mask=inputs.attention_mask	)[0].mean(axis=-1	)
									self.assertTrue(jnp.allclose(_UpperCAmelCase			,_UpperCAmelCase			,atol=4E-2	)	)

@require_flax
@slow
class 			__magic_name__		(					unittest.TestCase     ):







							@property
							def 							__lowercase    (							self      : Optional[int]	):
									return [
									    "Today is a beautiful day and I want",
									    "In the city of",
									    "Paris is the capital of France and",
									    "Computers and mobile phones have taken",
									]







							def 							__lowercase    (							self      : Any	):
									_a  :   Optional[Any]				 =							'facebook/opt-125m'

									_a  :   Optional[Any]				 =							[
									    'Today is a beautiful day and I want to',
									    'In the city of New York, the city',
									    'Paris is the capital of France and the capital',
									    'Computers and mobile phones have taken over the',
									]

									_a  :   Dict				 =							[]

									_a  :   Tuple				 =							FlaxOPTForCausalLM.from_pretrained(_UpperCAmelCase	)
									_a  :   Union[str, Any]				 =							GPTaTokenizer.from_pretrained(_UpperCAmelCase	)

									for prompt in self.prompts:
											_a  :   int				 =							tokenizer(_UpperCAmelCase			,return_tensors='jax'	).input_ids

											_a  :   int				 =							model.generate(_UpperCAmelCase			,max_length=10	)
											_a  :   Optional[Any]				 =							generated_ids[0]

											_a  :   List[Any]				 =							tokenizer.batch_decode(_UpperCAmelCase			,skip_special_tokens=_UpperCAmelCase	)
											predicted_outputs += generated_string

									self.assertListEqual(_UpperCAmelCase			,_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[str]	):
									_a  :   Tuple				 =							'facebook/opt-350m'

									_a  :   Dict				 =							[
									    'Today is a beautiful day and I want to',
									    'In the city of San Francisco, the city',
									    'Paris is the capital of France and the capital',
									    'Computers and mobile phones have taken over the',
									]

									_a  :   List[Any]				 =							[]
									_a  :   Any				 =							FlaxOPTForCausalLM.from_pretrained(_UpperCAmelCase	)
									_a  :   Any				 =							GPTaTokenizer.from_pretrained(_UpperCAmelCase	)

									for prompt in self.prompts:
											_a  :   List[Any]				 =							tokenizer(_UpperCAmelCase			,return_tensors='jax'	).input_ids

											_a  :   str				 =							model.generate(_UpperCAmelCase			,max_length=10	)
											_a  :   List[str]				 =							generated_ids[0]

											_a  :   Tuple				 =							tokenizer.batch_decode(_UpperCAmelCase			,skip_special_tokens=_UpperCAmelCase	)
											predicted_outputs += generated_string

									self.assertListEqual(_UpperCAmelCase			,_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[str]	):
									_a  :   List[str]				 =							'facebook/opt-125m'
									_a  :   Dict				 =							[
									    'Today is a beautiful day and I want to thank',
									    'In the city of Rome Canaver Canaver Canaver Canaver',
									]
									_a  :   int				 =							FlaxOPTForCausalLM.from_pretrained(_UpperCAmelCase	)
									_a  :   Dict				 =							GPTaTokenizer.from_pretrained(_UpperCAmelCase	)
									_a  :   List[Any]				 =							tokenizer(
									    [
									        'Today is a beautiful day and I want to',
									        'In the city of',
									    ]			,return_tensors='jax'			,padding=_UpperCAmelCase			,)

									_a  :   List[Any]				 =							jax.jit(model.generate	)

									_a  :   List[str]				 =							jit_generate(inputs['input_ids']			,attention_mask=inputs['attention_mask']	).sequences

									_a  :   int				 =							tokenizer.batch_decode(_UpperCAmelCase			,skip_special_tokens=_UpperCAmelCase	)

									self.assertIsNotNone(_UpperCAmelCase			,_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Tuple				 =							'facebook/opt-350m'

									_a  :   Union[str, Any]				 =							GPTaTokenizer.from_pretrained(_UpperCAmelCase	)
									_a  :   List[Any]				 =							FlaxOPTForCausalLM.from_pretrained(_UpperCAmelCase	)

									_a  :   Union[str, Any]				 =							'left'

									# use different length sentences to test batching
									_a  :   str				 =							[
									    'Hello, my dog is a little',
									    'Today, I',
									]

									_a  :   Tuple				 =							tokenizer(_UpperCAmelCase			,return_tensors='jax'			,padding=_UpperCAmelCase	)
									_a  :   Optional[Any]				 =							inputs['input_ids']

									_a  :   int				 =							model.generate(input_ids=_UpperCAmelCase			,attention_mask=inputs['attention_mask']			,trace=_UpperCAmelCase	)

									_a  :   List[str]				 =							tokenizer(sentences[0]			,return_tensors='jax'	).input_ids
									_a  :   Dict				 =							model.generate(input_ids=_UpperCAmelCase	)

									_a  :   List[Any]				 =							inputs_non_padded.shape[-1] - inputs['attention_mask'][-1].sum()
									_a  :   Optional[int]				 =							tokenizer(sentences[1]			,return_tensors='jax'	).input_ids
									_a  :   Optional[Any]				 =							model.generate(input_ids=_UpperCAmelCase			,max_length=model.config.max_length - num_paddings	)

									_a  :   Any				 =							tokenizer.batch_decode(outputs[0]			,skip_special_tokens=_UpperCAmelCase	)
									_a  :   Optional[Any]				 =							tokenizer.decode(output_non_padded[0][0]			,skip_special_tokens=_UpperCAmelCase	)
									_a  :   Optional[Any]				 =							tokenizer.decode(output_padded[0][0]			,skip_special_tokens=_UpperCAmelCase	)

									_a  :   Union[str, Any]				 =							[
									    'Hello, my dog is a little bit of a dork.\nI\'m a little bit',
									    'Today, I was in the middle of a conversation with a friend about the',
									]
									self.assertListEqual(_UpperCAmelCase			,_UpperCAmelCase	)
									self.assertListEqual(_UpperCAmelCase			,[non_padded_sentence, padded_sentence]	)









'''simple docstring'''

import unittest

from transformers import FunnelConfig, FunnelTokenizer, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MODEL_FOR_PRETRAINING_MAPPING,
							    FunnelBaseModel,
							    FunnelForMaskedLM,
							    FunnelForMultipleChoice,
							    FunnelForPreTraining,
							    FunnelForQuestionAnswering,
							    FunnelForSequenceClassification,
							    FunnelForTokenClassification,
							    FunnelModel,
							)

class 			__magic_name__		:







							def __init__(							self      : Tuple			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : int=13			,_UpperCAmelCase      : List[str]=7			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : List[Any]=True			,_UpperCAmelCase      : Tuple=True			,_UpperCAmelCase      : Optional[Any]=True			,_UpperCAmelCase      : str=99			,_UpperCAmelCase      : Optional[Any]=[1, 1, 2]			,_UpperCAmelCase      : Optional[Any]=1			,_UpperCAmelCase      : Any=32			,_UpperCAmelCase      : Optional[Any]=4			,_UpperCAmelCase      : Optional[int]=8			,_UpperCAmelCase      : str=37			,_UpperCAmelCase      : Tuple="gelu_new"			,_UpperCAmelCase      : Optional[int]=0.1			,_UpperCAmelCase      : List[Any]=0.1			,_UpperCAmelCase      : Dict=0.0			,_UpperCAmelCase      : str=512			,_UpperCAmelCase      : Any=3			,_UpperCAmelCase      : int=0.02			,_UpperCAmelCase      : str=3			,_UpperCAmelCase      : Tuple=4			,_UpperCAmelCase      : Optional[Any]=None			,_UpperCAmelCase      : Optional[int]=False			,):
									_a  :   Optional[Any]				 =							parent
									_a  :   Optional[Any]				 =							batch_size
									_a  :   Union[str, Any]				 =							seq_length
									_a  :   int				 =							is_training
									_a  :   List[Any]				 =							use_input_mask
									_a  :   Dict				 =							use_token_type_ids
									_a  :   List[str]				 =							use_labels
									_a  :   int				 =							vocab_size
									_a  :   Union[str, Any]				 =							block_sizes
									_a  :   int				 =							num_decoder_layers
									_a  :   List[str]				 =							d_model
									_a  :   Optional[Any]				 =							n_head
									_a  :   int				 =							d_head
									_a  :   Union[str, Any]				 =							d_inner
									_a  :   Any				 =							hidden_act
									_a  :   int				 =							hidden_dropout
									_a  :   str				 =							attention_dropout
									_a  :   Any				 =							activation_dropout
									_a  :   List[Any]				 =							max_position_embeddings
									_a  :   Optional[int]				 =							type_vocab_size
									_a  :   Optional[int]				 =							2
									_a  :   Dict				 =							num_labels
									_a  :   List[str]				 =							num_choices
									_a  :   List[str]				 =							scope
									_a  :   Any				 =							initializer_std

									# Used in the tests to check the size of the first attention layer
									_a  :   List[Any]				 =							n_head
									# Used in the tests to check the size of the first hidden state
									_a  :   List[Any]				 =							self.d_model
									# Used in the tests to check the number of output hidden states/attentions
									_a  :   Tuple				 =							sum(self.block_sizes	) + (0 if base else self.num_decoder_layers)
									# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
									# the last hidden state of the first block (which is the first hidden state of the decoder).
									if not base:
											_a  :   Tuple				 =							self.num_hidden_layers + 2







							def 							__lowercase    (							self      : Any	):
									_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.vocab_size	)

									_a  :   Dict				 =							None
									if self.use_input_mask:
											_a  :   str				 =							ids_tensor([self.batch_size, self.seq_length]			,vocab_size=2	)

									_a  :   Any				 =							None
									if self.use_token_type_ids:
											_a  :   Optional[int]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.type_vocab_size	)

									_a  :   str				 =							None
									_a  :   Optional[int]				 =							None
									_a  :   str				 =							None
									if self.use_labels:
											_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size]			,self.type_sequence_label_size	)
											_a  :   List[Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,self.num_labels	)
											_a  :   List[str]				 =							ids_tensor([self.batch_size]			,self.num_choices	)
											_a  :   Union[str, Any]				 =							ids_tensor([self.batch_size, self.seq_length]			,1	)

									_a  :   Dict				 =							self.get_config()

									return (
									    config,
									    input_ids,
									    token_type_ids,
									    input_mask,
									    sequence_labels,
									    token_labels,
									    choice_labels,
									    fake_token_labels,
									)







							def 							__lowercase    (							self      : Any	):
									return FunnelConfig(
									    vocab_size=self.vocab_size			,block_sizes=self.block_sizes			,num_decoder_layers=self.num_decoder_layers			,d_model=self.d_model			,n_head=self.n_head			,d_head=self.d_head			,d_inner=self.d_inner			,hidden_act=self.hidden_act			,hidden_dropout=self.hidden_dropout			,attention_dropout=self.attention_dropout			,activation_dropout=self.activation_dropout			,max_position_embeddings=self.max_position_embeddings			,type_vocab_size=self.type_vocab_size			,initializer_std=self.initializer_std			,)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : List[str]			,):
									_a  :   int				 =							FunnelModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   int				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   List[str]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   int				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)

									_a  :   Optional[Any]				 =							False
									_a  :   str				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)

									_a  :   Tuple				 =							False
									_a  :   Dict				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, self.seq_length, self.d_model)	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Any			,):
									_a  :   int				 =							FunnelBaseModel(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Optional[Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   int				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)
									_a  :   Tuple				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 2, self.d_model)	)

									_a  :   Dict				 =							False
									_a  :   Dict				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 3, self.d_model)	)

									_a  :   Tuple				 =							False
									_a  :   List[Any]				 =							model(_UpperCAmelCase	)
									self.parent.assertEqual(result.last_hidden_state.shape			,(self.batch_size, 2, self.d_model)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any			,):
									_a  :   Union[str, Any]				 =							self.num_labels
									_a  :   Any				 =							FunnelForPreTraining(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length)	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : List[Any]			,):
									_a  :   Tuple				 =							FunnelForMaskedLM(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   List[str]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.vocab_size)	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[int]			,):
									_a  :   List[str]				 =							self.num_labels
									_a  :   Dict				 =							FunnelForSequenceClassification(_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   str				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_labels)	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : int			,_UpperCAmelCase      : Union[str, Any]			,_UpperCAmelCase      : List[Any]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : int			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Optional[int]			,):
									_a  :   int				 =							self.num_choices
									_a  :   Dict				 =							FunnelForMultipleChoice(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Any				 =							input_ids.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   List[Any]				 =							token_type_ids.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   Tuple				 =							input_mask.unsqueeze(1	).expand(-1			,self.num_choices			,-1	).contiguous()
									_a  :   Union[str, Any]				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase			,)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.num_choices)	)







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : Optional[int]			,_UpperCAmelCase      : List[str]			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Any			,):
									_a  :   Any				 =							self.num_labels
									_a  :   int				 =							FunnelForTokenClassification(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Union[str, Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,labels=_UpperCAmelCase	)
									self.parent.assertEqual(result.logits.shape			,(self.batch_size, self.seq_length, self.num_labels)	)







							def 							__lowercase    (							self      : int			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Tuple			,_UpperCAmelCase      : str			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Dict			,_UpperCAmelCase      : Optional[Any]			,):
									_a  :   List[str]				 =							FunnelForQuestionAnswering(config=_UpperCAmelCase	)
									model.to(_UpperCAmelCase	)
									model.eval()
									_a  :   Union[str, Any]				 =							model(
									    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase			,start_positions=_UpperCAmelCase			,end_positions=_UpperCAmelCase			,)
									self.parent.assertEqual(result.start_logits.shape			,(self.batch_size, self.seq_length)	)
									self.parent.assertEqual(result.end_logits.shape			,(self.batch_size, self.seq_length)	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Tuple				 =							self.prepare_config_and_inputs()
									(
									    (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      (
									    _a
									) ,      
									)  :   List[str]				 =							config_and_inputs
									_a  :   int				 =							{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
									return config, inputs_dict

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					int            =      False
							lowerCAmelCase :					List[Any]            =      False
							lowerCAmelCase :					Union[str, Any]            =      (
							    (
							        FunnelModel,
							        FunnelForMaskedLM,
							        FunnelForPreTraining,
							        FunnelForQuestionAnswering,
							        FunnelForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							lowerCAmelCase :					Optional[Any]            =      (
							    {
							        'feature-extraction': (FunnelBaseModel, FunnelModel),
							        'fill-mask': FunnelForMaskedLM,
							        'question-answering': FunnelForQuestionAnswering,
							        'text-classification': FunnelForSequenceClassification,
							        'token-classification': FunnelForTokenClassification,
							        'zero-shot': FunnelForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : Optional[Any]			,_UpperCAmelCase      : Any			,_UpperCAmelCase      : Optional[int]=False	):
									_a  :   List[Any]				 =							super()._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase			,return_labels=_UpperCAmelCase	)

									if return_labels:
											if model_class in get_values(_UpperCAmelCase	):
													_a  :   Optional[int]				 =							torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.seq_length)			,dtype=torch.long			,device=_UpperCAmelCase	)
									return inputs_dict







							def 							__lowercase    (							self      : List[Any]	):
									_a  :   int				 =							FunnelModelTester(self	)
									_a  :   List[str]				 =							ConfigTester(self			,config_class=_UpperCAmelCase	)







							def 							__lowercase    (							self      : Tuple	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : Any	):
									_a  :   Any				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Union[str, Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_pretraining(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Tuple				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_masked_lm(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Any	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Optional[int]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[Any]			,_UpperCAmelCase      : Union[str, Any]	):
									if hasattr(_UpperCAmelCase			,'weight'	) and module.weight is not None:
											module.weight.data.fill_(3	)
									if hasattr(_UpperCAmelCase			,'bias'	) and module.bias is not None:
											module.bias.data.fill_(3	)

									for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
											if hasattr(_UpperCAmelCase			,_UpperCAmelCase	) and getattr(_UpperCAmelCase			,_UpperCAmelCase	) is not None:
													_a  :   str				 =							getattr(_UpperCAmelCase			,_UpperCAmelCase	)
													weight.data.fill_(3	)

@require_torch
class 			__magic_name__		(					_UpperCamelCase    ,		unittest.TestCase     ):
							lowerCAmelCase :					Optional[int]            =      False
							lowerCAmelCase :					Optional[Any]            =      False
							lowerCAmelCase :					Optional[Any]            =      (
							    (FunnelBaseModel, FunnelForMultipleChoice, FunnelForSequenceClassification) if is_torch_available() else ()
							)







							def 							__lowercase    (							self      : int	):
									_a  :   Dict				 =							FunnelModelTester(self			,base=_UpperCAmelCase	)
									_a  :   Union[str, Any]				 =							ConfigTester(self			,config_class=_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[int]	):
									self.config_tester.run_common_tests()







							def 							__lowercase    (							self      : Tuple	):
									_a  :   Optional[Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_base_model(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Dict				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : Optional[Any]	):
									_a  :   Optional[Any]				 =							self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_multiple_choice(*_UpperCAmelCase	)







							def 							__lowercase    (							self      : List[str]	):
									_a ,      _a  :   Optional[int]				 =							self.model_tester.prepare_config_and_inputs_for_common()
									_a  :   str				 =							True

									for model_class in self.all_model_classes:
											if model_class.__name__ == "FunnelBaseModel":
													continue
											_a  :   Optional[int]				 =							model_class(_UpperCAmelCase	)
											model.to(_UpperCAmelCase	)
											model.train()
											_a  :   Dict				 =							self._prepare_for_class(_UpperCAmelCase			,_UpperCAmelCase			,return_labels=_UpperCAmelCase	)
											_a  :   Dict				 =							model(**_UpperCAmelCase	).loss
											loss.backward()







							def 							__lowercase    (							self      : Optional[int]			,_UpperCAmelCase      : List[Any]	):
									if hasattr(_UpperCAmelCase			,'weight'	) and module.weight is not None:
											module.weight.data.fill_(3	)
									if hasattr(_UpperCAmelCase			,'bias'	) and module.bias is not None:
											module.bias.data.fill_(3	)

									for param in ["r_w_bias", "r_r_bias", "r_kernel", "r_s_bias", "seg_embed"]:
											if hasattr(_UpperCAmelCase			,_UpperCAmelCase	) and getattr(_UpperCAmelCase			,_UpperCAmelCase	) is not None:
													_a  :   Tuple				 =							getattr(_UpperCAmelCase			,_UpperCAmelCase	)
													weight.data.fill_(3	)

@require_torch
@require_sentencepiece
@require_tokenizers
class 			__magic_name__		(					unittest.TestCase     ):







							def 							__lowercase    (							self      : Any	):
									_a  :   List[Any]				 =							13
									_a  :   Dict				 =							7
									_a  :   Union[str, Any]				 =							torch.arange(0			,batch_size * sequence_length	).long().reshape(_UpperCAmelCase			,_UpperCAmelCase	)
									_a  :   Tuple				 =							[0, 1, 2, 3, 4, 5, 6, 4, 1, 3, 5, 0, 1]
									_a  :   List[Any]				 =							torch.tensor([[2] + [0] * a + [1] * (sequence_length - a - 1) for a in lengths]	)

									_a  :   Tuple				 =							FunnelModel.from_pretrained('sgugger/funnel-random-tiny'	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)[0].abs()

									_a  :   Optional[int]				 =							torch.tensor(23_44.83_52	)
									_a  :   Union[str, Any]				 =							torch.tensor(0.80_52	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)

									_a  :   Tuple				 =							torch.tensor([[1] * 7, [1] * 4 + [0] * 3] * 6 + [[0, 1, 1, 0, 0, 1, 1]]	)
									_a  :   List[Any]				 =							model(_UpperCAmelCase			,attention_mask=_UpperCAmelCase			,token_type_ids=_UpperCAmelCase	)[0].abs()

									_a  :   Union[str, Any]				 =							torch.tensor(23_43.84_25	)
									_a  :   Tuple				 =							torch.tensor(0.80_49	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)







							@slow
							def 							__lowercase    (							self      : str	):
									_a  :   Union[str, Any]				 =							FunnelTokenizer.from_pretrained('huggingface/funnel-small'	)
									_a  :   Tuple				 =							FunnelModel.from_pretrained('huggingface/funnel-small'	)
									_a  :   Union[str, Any]				 =							tokenizer('Hello! I am the Funnel Transformer model.'			,return_tensors='pt'	)
									_a  :   str				 =							model(**_UpperCAmelCase	)[0]

									_a  :   str				 =							torch.tensor(2_35.72_46	)
									_a  :   List[str]				 =							torch.tensor(0.02_56	)
									self.assertTrue(torch.allclose(output.sum()			,_UpperCAmelCase			,atol=1E-4	)	)
									self.assertTrue(torch.allclose(output.mean()			,_UpperCAmelCase			,atol=1E-4	)	)




