





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    RemBertConfig,
		    RemBertForCausalLM,
		    RemBertForMaskedLM,
		    RemBertForMultipleChoice,
		    RemBertForQuestionAnswering,
		    RemBertForSequenceClassification,
		    RemBertForTokenClassification,
		    RemBertModel,
		)
		from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST






class 	UpperCamelCase_     :



					def __init__(		self    :    int					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    Dict=13					,			lowerCamelCase    :    Union[str, Any]=7					,			lowerCamelCase    :    Any=True					,			lowerCamelCase    :    int=True					,			lowerCamelCase    :    int=True					,			lowerCamelCase    :    int=True					,			lowerCamelCase    :    Union[str, Any]=99					,			lowerCamelCase    :    Any=32					,			lowerCamelCase    :    int=18					,			lowerCamelCase    :    Optional[Any]=43					,			lowerCamelCase    :    int=5					,			lowerCamelCase    :    str=4					,			lowerCamelCase    :    Tuple=37					,			lowerCamelCase    :    Any="gelu"					,			lowerCamelCase    :    Optional[int]=0.1					,			lowerCamelCase    :    Optional[Any]=0.1					,			lowerCamelCase    :    Optional[Any]=5_12					,			lowerCamelCase    :    Optional[int]=16					,			lowerCamelCase    :    Any=2					,			lowerCamelCase    :    Union[str, Any]=0.02					,			lowerCamelCase    :    Union[str, Any]=3					,			lowerCamelCase    :    Tuple=4					,			lowerCamelCase    :    int=None					,			):
							lowerCamelCase_  :    int   	=			parent
							lowerCamelCase_  :    int   	=			batch_size
							lowerCamelCase_  :    List[str]   	=			seq_length
							lowerCamelCase_  :    Optional[int]   	=			is_training
							lowerCamelCase_  :    int   	=			use_input_mask
							lowerCamelCase_  :    List[str]   	=			use_token_type_ids
							lowerCamelCase_  :    Union[str, Any]   	=			use_labels
							lowerCamelCase_  :    Tuple   	=			vocab_size
							lowerCamelCase_  :    int   	=			hidden_size
							lowerCamelCase_  :    Tuple   	=			input_embedding_size
							lowerCamelCase_  :    Any   	=			output_embedding_size
							lowerCamelCase_  :    Tuple   	=			num_hidden_layers
							lowerCamelCase_  :    Dict   	=			num_attention_heads
							lowerCamelCase_  :    Union[str, Any]   	=			intermediate_size
							lowerCamelCase_  :    int   	=			hidden_act
							lowerCamelCase_  :    str   	=			hidden_dropout_prob
							lowerCamelCase_  :    str   	=			attention_probs_dropout_prob
							lowerCamelCase_  :    Tuple   	=			max_position_embeddings
							lowerCamelCase_  :    Optional[Any]   	=			type_vocab_size
							lowerCamelCase_  :    List[str]   	=			type_sequence_label_size
							lowerCamelCase_  :    int   	=			initializer_range
							lowerCamelCase_  :    List[str]   	=			num_labels
							lowerCamelCase_  :    List[Any]   	=			num_choices
							lowerCamelCase_  :    Tuple   	=			scope



					def      __a    (		self    :    str	):
							lowerCamelCase_  :    Union[str, Any]   	=			ids_tensor([self.batch_size, self.seq_length]					,			self.vocab_size	)

							lowerCamelCase_  :    int   	=			None
							if self.use_input_mask:
									lowerCamelCase_  :    str   	=			random_attention_mask([self.batch_size, self.seq_length]	)

							lowerCamelCase_  :    str   	=			None
							if self.use_token_type_ids:
									lowerCamelCase_  :    str   	=			ids_tensor([self.batch_size, self.seq_length]					,			self.type_vocab_size	)

							lowerCamelCase_  :    Union[str, Any]   	=			None
							lowerCamelCase_  :    Union[str, Any]   	=			None
							lowerCamelCase_  :    List[str]   	=			None
							if self.use_labels:
									lowerCamelCase_  :    str   	=			ids_tensor([self.batch_size]					,			self.type_sequence_label_size	)
									lowerCamelCase_  :    str   	=			ids_tensor([self.batch_size, self.seq_length]					,			self.num_labels	)
									lowerCamelCase_  :    List[str]   	=			ids_tensor([self.batch_size]					,			self.num_choices	)

							lowerCamelCase_  :    Optional[int]   	=			RemBertConfig(
							    vocab_size=self.vocab_size					,			hidden_size=self.hidden_size					,			input_embedding_size=self.input_embedding_size					,			output_embedding_size=self.output_embedding_size					,			num_hidden_layers=self.num_hidden_layers					,			num_attention_heads=self.num_attention_heads					,			intermediate_size=self.intermediate_size					,			hidden_act=self.hidden_act					,			hidden_dropout_prob=self.hidden_dropout_prob					,			attention_probs_dropout_prob=self.attention_probs_dropout_prob					,			max_position_embeddings=self.max_position_embeddings					,			type_vocab_size=self.type_vocab_size					,			is_decoder=lowerCamelCase					,			initializer_range=self.initializer_range					,			)

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



					def      __a    (		self    :    Optional[Any]	):
							(
							    (
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			
							)  :    int   	=			self.prepare_config_and_inputs()

							lowerCamelCase_  :    List[Any]   	=			True
							lowerCamelCase_  :    Optional[int]   	=			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
							lowerCamelCase_  :    Optional[Any]   	=			ids_tensor([self.batch_size, self.seq_length]					,			vocab_size=2	)

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_mask,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							    encoder_hidden_states,
							    encoder_attention_mask,
							)



					def      __a    (		self    :    str					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Optional[int]	):
							lowerCamelCase_  :    str   	=			RemBertModel(config=lowerCamelCase	)
							model.to(lowerCamelCase	)
							model.eval()
							lowerCamelCase_  :    Any   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase	)
							lowerCamelCase_  :    str   	=			model(lowerCamelCase					,			token_type_ids=lowerCamelCase	)
							lowerCamelCase_  :    Union[str, Any]   	=			model(lowerCamelCase	)
							self.parent.assertEqual(result.last_hidden_state.shape					,			(self.batch_size, self.seq_length, self.hidden_size)	)



					def      __a    (		self    :    List[str]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    List[str]					,			):
							lowerCamelCase_  :    Optional[int]   	=			True
							lowerCamelCase_  :    Union[str, Any]   	=			RemBertModel(lowerCamelCase	)
							model.to(lowerCamelCase	)
							model.eval()
							lowerCamelCase_  :    Tuple   	=			model(
							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			encoder_attention_mask=lowerCamelCase					,			)
							lowerCamelCase_  :    int   	=			model(
							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			)
							lowerCamelCase_  :    int   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase	)
							self.parent.assertEqual(result.last_hidden_state.shape					,			(self.batch_size, self.seq_length, self.hidden_size)	)



					def      __a    (		self    :    int					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    str					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Optional[Any]					,			):
							lowerCamelCase_  :    Any   	=			RemBertForCausalLM(config=lowerCamelCase	)
							model.to(lowerCamelCase	)
							model.eval()
							lowerCamelCase_  :    Union[str, Any]   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
							self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.vocab_size)	)



					def      __a    (		self    :    Any					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    List[Any]	):
							lowerCamelCase_  :    str   	=			RemBertForMaskedLM(config=lowerCamelCase	)
							model.to(lowerCamelCase	)
							model.eval()
							lowerCamelCase_  :    int   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
							self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.vocab_size)	)



					def      __a    (		self    :    int					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    List[str]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Union[str, Any]					,			lowerCamelCase    :    Dict					,			):
							lowerCamelCase_  :    List[Any]   	=			True
							lowerCamelCase_  :    Tuple   	=			True
							lowerCamelCase_  :    List[str]   	=			RemBertForCausalLM(config=lowerCamelCase	)
							model.to(lowerCamelCase	)
							model.eval()

							# first forward pass
							lowerCamelCase_  :    Optional[int]   	=			model(
							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			encoder_attention_mask=lowerCamelCase					,			use_cache=lowerCamelCase					,			)
							lowerCamelCase_  :    List[str]   	=			outputs.past_key_values

							# create hypothetical multiple next token and extent to next_input_ids
							lowerCamelCase_  :    List[str]   	=			ids_tensor((self.batch_size, 3)					,			config.vocab_size	)
							lowerCamelCase_  :    Dict   	=			ids_tensor((self.batch_size, 3)					,			vocab_size=2	)

							# append to next input_ids and
							lowerCamelCase_  :    Dict   	=			torch.cat([input_ids, next_tokens]					,			dim=-1	)
							lowerCamelCase_  :    List[Any]   	=			torch.cat([input_mask, next_mask]					,			dim=-1	)

							lowerCamelCase_  :    Optional[Any]   	=			model(
							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			encoder_attention_mask=lowerCamelCase					,			output_hidden_states=lowerCamelCase					,			)['hidden_states'][0]
							lowerCamelCase_  :    Union[str, Any]   	=			model(
							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			encoder_hidden_states=lowerCamelCase					,			encoder_attention_mask=lowerCamelCase					,			past_key_values=lowerCamelCase					,			output_hidden_states=lowerCamelCase					,			)['hidden_states'][0]

							# select random slice
							lowerCamelCase_  :    int   	=			ids_tensor((1,)					,			output_from_past.shape[-1]	).item()
							lowerCamelCase_  :    str   	=			output_from_no_past[:, -3:, random_slice_idx].detach()
							lowerCamelCase_  :    List[Any]   	=			output_from_past[:, :, random_slice_idx].detach()

							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(lowerCamelCase					,			lowerCamelCase					,			atol=1E-3	)	)



					def      __a    (		self    :    List[Any]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    str					,			lowerCamelCase    :    str					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Optional[int]	):
							lowerCamelCase_  :    Dict   	=			RemBertForQuestionAnswering(config=lowerCamelCase	)
							model.to(lowerCamelCase	)
							model.eval()
							lowerCamelCase_  :    List[Any]   	=			model(
							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			start_positions=lowerCamelCase					,			end_positions=lowerCamelCase					,			)
							self.parent.assertEqual(result.start_logits.shape					,			(self.batch_size, self.seq_length)	)
							self.parent.assertEqual(result.end_logits.shape					,			(self.batch_size, self.seq_length)	)



					def      __a    (		self    :    int					,			lowerCamelCase    :    str					,			lowerCamelCase    :    str					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    List[Any]	):
							lowerCamelCase_  :    Optional[Any]   	=			self.num_labels
							lowerCamelCase_  :    Union[str, Any]   	=			RemBertForSequenceClassification(lowerCamelCase	)
							model.to(lowerCamelCase	)
							model.eval()
							lowerCamelCase_  :    Any   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
							self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.num_labels)	)



					def      __a    (		self    :    Any					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Optional[Any]					,			lowerCamelCase    :    Optional[int]					,			lowerCamelCase    :    Tuple					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    List[str]	):
							lowerCamelCase_  :    Tuple   	=			self.num_labels
							lowerCamelCase_  :    Tuple   	=			RemBertForTokenClassification(config=lowerCamelCase	)
							model.to(lowerCamelCase	)
							model.eval()
							lowerCamelCase_  :    Optional[int]   	=			model(lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase	)
							self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.seq_length, self.num_labels)	)



					def      __a    (		self    :    str					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    List[Any]					,			lowerCamelCase    :    Any					,			lowerCamelCase    :    Dict					,			lowerCamelCase    :    Tuple	):
							lowerCamelCase_  :    Any   	=			self.num_choices
							lowerCamelCase_  :    Dict   	=			RemBertForMultipleChoice(config=lowerCamelCase	)
							model.to(lowerCamelCase	)
							model.eval()
							lowerCamelCase_  :    Any   	=			input_ids.unsqueeze(1	).expand(-1					,			self.num_choices					,			-1	).contiguous()
							lowerCamelCase_  :    List[Any]   	=			token_type_ids.unsqueeze(1	).expand(-1					,			self.num_choices					,			-1	).contiguous()
							lowerCamelCase_  :    List[Any]   	=			input_mask.unsqueeze(1	).expand(-1					,			self.num_choices					,			-1	).contiguous()
							lowerCamelCase_  :    str   	=			model(
							    lowerCamelCase					,			attention_mask=lowerCamelCase					,			token_type_ids=lowerCamelCase					,			labels=lowerCamelCase					,			)
							self.parent.assertEqual(result.logits.shape					,			(self.batch_size, self.num_choices)	)





					def      __a    (		self    :    Optional[Any]	):
							lowerCamelCase_  :    Dict   	=			self.prepare_config_and_inputs()
							(
							    (
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			
							)  :    int   	=			config_and_inputs
							lowerCamelCase_  :    Optional[Any]   	=			{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict






@require_torch
class 	UpperCamelCase_     (     snake_case__     ,    snake_case__     ,    unittest.TestCase       ):
					_a			:	int							=				(
					    (
					        RemBertModel,
					        RemBertForMaskedLM,
					        RemBertForCausalLM,
					        RemBertForMultipleChoice,
					        RemBertForQuestionAnswering,
					        RemBertForSequenceClassification,
					        RemBertForTokenClassification,
					    )
					    if is_torch_available()
					    else ()
					)
					_a			:	Tuple							=				(RemBertForCausalLM,) if is_torch_available() else ()
					_a			:	List[Any]							=				(
					    {
					        'feature-extraction': RemBertModel,
					        'fill-mask': RemBertForMaskedLM,
					        'question-answering': RemBertForQuestionAnswering,
					        'text-classification': RemBertForSequenceClassification,
					        'text-generation': RemBertForCausalLM,
					        'token-classification': RemBertForTokenClassification,
					        'zero-shot': RemBertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)



					def      __a    (		self    :    Dict	):
							lowerCamelCase_  :    Dict   	=			RemBertModelTester(self	)
							lowerCamelCase_  :    str   	=			ConfigTester(self					,			config_class=lowerCamelCase					,			hidden_size=37	)



					def      __a    (		self    :    Optional[int]	):
							self.config_tester.run_common_tests()



					def      __a    (		self    :    List[Any]	):
							lowerCamelCase_  :    List[str]   	=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*lowerCamelCase	)



					def      __a    (		self    :    Any	):
							lowerCamelCase_  :    Union[str, Any]   	=			self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
									lowerCamelCase_  :    Optional[Any]   	=			type
									self.model_tester.create_and_check_model(*lowerCamelCase	)



					def      __a    (		self    :    Dict	):
							lowerCamelCase_  :    Optional[Any]   	=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*lowerCamelCase	)



					def      __a    (		self    :    int	):
							lowerCamelCase_  :    Optional[Any]   	=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*lowerCamelCase	)



					def      __a    (		self    :    str	):
							lowerCamelCase_  :    Optional[Any]   	=			self.model_tester.prepare_config_and_inputs_for_decoder()
							self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCamelCase	)



					def      __a    (		self    :    int	):
							lowerCamelCase_  :    List[Any]   	=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*lowerCamelCase	)



					def      __a    (		self    :    str	):
							lowerCamelCase_  :    Optional[Any]   	=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase	)



					def      __a    (		self    :    int	):
							lowerCamelCase_  :    Any   	=			self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*lowerCamelCase	)



					def      __a    (		self    :    List[str]	):
							lowerCamelCase_  :    Optional[Any]   	=			self.model_tester.prepare_config_and_inputs_for_decoder()
							self.model_tester.create_and_check_model_as_decoder(*lowerCamelCase	)



					def      __a    (		self    :    str	):
							# This regression test was failing with PyTorch < 1.3
							(
							    (
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			(
							    lowerCamelCase_
							)	,			
							)  :    int   	=			self.model_tester.prepare_config_and_inputs_for_decoder()

							lowerCamelCase_  :    str   	=			None

							self.model_tester.create_and_check_model_as_decoder(
							    lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			)





					@slow
					def      __a    (		self    :    Any	):
							for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowerCamelCase_  :    int   	=			RemBertModel.from_pretrained(lowerCamelCase	)
									self.assertIsNotNone(lowerCamelCase	)






@require_torch
class 	UpperCamelCase_     (     unittest.TestCase       ):



					@slow
					def      __a    (		self    :    Dict	):
							# Test exact values at the last hidden layer
							lowerCamelCase_  :    Dict   	=			RemBertModel.from_pretrained('google/rembert'	)
							lowerCamelCase_  :    List[Any]   	=			torch.tensor([[3_12, 5_64_98, 3_13, 21_25, 3_13]]	)
							lowerCamelCase_  :    Any   	=			torch.tensor([[0, 0, 0, 1, 1]]	)
							with torch.no_grad():
									lowerCamelCase_  :    Union[str, Any]   	=			model(lowerCamelCase					,			token_type_ids=lowerCamelCase					,			output_hidden_states=lowerCamelCase	)

							lowerCamelCase_  :    Union[str, Any]   	=			11_52

							lowerCamelCase_  :    Optional[int]   	=			torch.Size((1, 5, hidden_size)	)
							self.assertEqual(output['last_hidden_state'].shape					,			lowerCamelCase	)

							lowerCamelCase_  :    Tuple   	=			torch.tensor(
							    [
							        [
							            [0.0_754, -0.2_022, 0.1_904],
							            [-0.3_354, -0.3_692, -0.4_791],
							            [-0.2_314, -0.6_729, -0.0_749],
							            [-0.0_396, -0.3_105, -0.4_234],
							            [-0.1_571, -0.0_525, 0.5_353],
							        ]
							    ]	)

							# Running on the original tf implementation gives slightly different results here.
							# Not clear why this variations is present
							# TODO: Find reason for discrepancy
							# expected_original_implementation = [[
							#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
							#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
							#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
							#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
							#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
							# ]]

							self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]					,			lowerCamelCase					,			atol=1E-4	)	)







from typing import Optional, Tuple, Union

import tensorflow as tf

from ...activations_tf import ACTaFN
from ...modeling_tf_outputs import (
    TFBaseModelOutputWithNoAttention,
    TFBaseModelOutputWithPoolingAndNoAttention,
    TFImageClassifierOutputWithNoAttention,
)
from ...modeling_tf_utils import TFPreTrainedModel, TFSequenceClassificationLoss, keras_serializable, unpack_inputs
from ...tf_utils import shape_list
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_resnet import ResNetConfig


_lowercase		:     Optional[Any] 					=logging.get_logger(__name__)

# General docstring
_lowercase		:     Union[str, Any] 					="""ResNetConfig"""

# Base docstring
_lowercase		:     Dict 					="""microsoft/resnet-50"""
_lowercase		:     Tuple 					=[1, 2048, 7, 7]

# Image classification docstring
_lowercase		:     Dict 					="""microsoft/resnet-50"""
_lowercase		:     int 					="""tiger cat"""

_lowercase		:     Optional[Any] 					=[
    """microsoft/resnet-50""",
    # See all resnet models at https://huggingface.co/models?filter=resnet
]






class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):



					def __init__(		self    :    List[str]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int = 3					,			lowerCamelCase    :    int = 1					,			lowerCamelCase    :    str = "relu"					,			**lowerCamelCase    :    Tuple	):
							super().__init__(**lowerCamelCase	)
							lowerCamelCase_  :    Tuple   	=			kernel_size // 2
							lowerCamelCase_  :    Optional[int]   	=			tf.keras.layers.ConvaD(
							    lowerCamelCase					,			kernel_size=lowerCamelCase					,			strides=lowerCamelCase					,			padding='valid'					,			use_bias=lowerCamelCase					,			name='convolution'	)
							# Use same default momentum and epsilon as PyTorch equivalent
							lowerCamelCase_  :    Union[str, Any]   	=			tf.keras.layers.BatchNormalization(epsilon=1E-5					,			momentum=0.9					,			name='normalization'	)
							lowerCamelCase_  :    Optional[Any]   	=			ACTaFN[activation] if activation is not None else tf.keras.layers.Activation('linear'	)



					def      __a    (		self    :    int					,			lowerCamelCase    :    tf.Tensor	):
							# Pad to match that done in the PyTorch Conv2D model
							lowerCamelCase_  :    int   	=			(self.pad_value, self.pad_value)
							lowerCamelCase_  :    Optional[Any]   	=			tf.pad(lowerCamelCase					,			[(0, 0), height_pad, width_pad, (0, 0)]	)
							lowerCamelCase_  :    str   	=			self.conv(lowerCamelCase	)
							return hidden_state





					def      __a    (		self    :    Tuple					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
							lowerCamelCase_  :    Optional[Any]   	=			self.convolution(lowerCamelCase	)
							lowerCamelCase_  :    int   	=			self.normalization(lowerCamelCase					,			training=lowerCamelCase	)
							lowerCamelCase_  :    Any   	=			self.activation(lowerCamelCase	)
							return hidden_state






class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):



					def __init__(		self    :    Dict					,			lowerCamelCase    :    ResNetConfig					,			**lowerCamelCase    :    Optional[Any]	):
							super().__init__(**lowerCamelCase	)
							lowerCamelCase_  :    Union[str, Any]   	=			TFResNetConvLayer(
							    config.embedding_size					,			kernel_size=7					,			stride=2					,			activation=config.hidden_act					,			name='embedder'					,			)
							lowerCamelCase_  :    Any   	=			tf.keras.layers.MaxPoolaD(pool_size=3					,			strides=2					,			padding='valid'					,			name='pooler'	)
							lowerCamelCase_  :    Any   	=			config.num_channels





					def      __a    (		self    :    List[Any]					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
							lowerCamelCase_	,			lowerCamelCase_	,			lowerCamelCase_	,			lowerCamelCase_  :    Tuple   	=			shape_list(lowerCamelCase	)
							if tf.executing_eagerly() and num_channels != self.num_channels:
									raise ValueError(
									    'Make sure that the channel dimension of the pixel values match with the one set in the configuration.'	)
							lowerCamelCase_  :    Optional[int]   	=			pixel_values
							lowerCamelCase_  :    Optional[Any]   	=			self.embedder(lowerCamelCase	)
							lowerCamelCase_  :    Optional[int]   	=			tf.pad(lowerCamelCase					,			[[0, 0], [1, 1], [1, 1], [0, 0]]	)
							lowerCamelCase_  :    str   	=			self.pooler(lowerCamelCase	)
							return hidden_state






class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):



					def __init__(		self    :    Union[str, Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int = 2					,			**lowerCamelCase    :    List[Any]	):
							super().__init__(**lowerCamelCase	)
							lowerCamelCase_  :    int   	=			tf.keras.layers.ConvaD(
							    lowerCamelCase					,			kernel_size=1					,			strides=lowerCamelCase					,			use_bias=lowerCamelCase					,			name='convolution'	)
							# Use same default momentum and epsilon as PyTorch equivalent
							lowerCamelCase_  :    Any   	=			tf.keras.layers.BatchNormalization(epsilon=1E-5					,			momentum=0.9					,			name='normalization'	)





					def      __a    (		self    :    Tuple					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
							lowerCamelCase_  :    Union[str, Any]   	=			x
							lowerCamelCase_  :    Optional[int]   	=			self.convolution(lowerCamelCase	)
							lowerCamelCase_  :    List[str]   	=			self.normalization(lowerCamelCase					,			training=lowerCamelCase	)
							return hidden_state






class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):



					def __init__(		self    :    List[Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int = 1					,			lowerCamelCase    :    str = "relu"					,			**lowerCamelCase    :    Optional[int]	):
							super().__init__(**lowerCamelCase	)
							lowerCamelCase_  :    Any   	=			in_channels != out_channels or stride != 1
							lowerCamelCase_  :    Tuple   	=			TFResNetConvLayer(lowerCamelCase					,			stride=lowerCamelCase					,			name='layer.0'	)
							lowerCamelCase_  :    str   	=			TFResNetConvLayer(lowerCamelCase					,			activation=lowerCamelCase					,			name='layer.1'	)
							lowerCamelCase_  :    Any   	=			(
							    TFResNetShortCut(lowerCamelCase					,			stride=lowerCamelCase					,			name='shortcut'	)
							    if should_apply_shortcut
							    else tf.keras.layers.Activation('linear'					,			name='shortcut'	)
							)
							lowerCamelCase_  :    str   	=			ACTaFN[activation]





					def      __a    (		self    :    List[Any]					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
							lowerCamelCase_  :    Optional[int]   	=			hidden_state
							lowerCamelCase_  :    Tuple   	=			self.conva(lowerCamelCase					,			training=lowerCamelCase	)
							lowerCamelCase_  :    List[Any]   	=			self.conva(lowerCamelCase					,			training=lowerCamelCase	)
							lowerCamelCase_  :    int   	=			self.shortcut(lowerCamelCase					,			training=lowerCamelCase	)
							hidden_state += residual
							lowerCamelCase_  :    Dict   	=			self.activation(lowerCamelCase	)
							return hidden_state






class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):



					def __init__(		self    :    Optional[Any]					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int = 1					,			lowerCamelCase    :    str = "relu"					,			lowerCamelCase    :    int = 4					,			**lowerCamelCase    :    Optional[int]					,			):
							super().__init__(**lowerCamelCase	)
							lowerCamelCase_  :    List[Any]   	=			in_channels != out_channels or stride != 1
							lowerCamelCase_  :    Optional[Any]   	=			out_channels // reduction
							lowerCamelCase_  :    int   	=			TFResNetConvLayer(lowerCamelCase					,			kernel_size=1					,			name='layer.0'	)
							lowerCamelCase_  :    Tuple   	=			TFResNetConvLayer(lowerCamelCase					,			stride=lowerCamelCase					,			name='layer.1'	)
							lowerCamelCase_  :    Optional[int]   	=			TFResNetConvLayer(lowerCamelCase					,			kernel_size=1					,			activation=lowerCamelCase					,			name='layer.2'	)
							lowerCamelCase_  :    Optional[Any]   	=			(
							    TFResNetShortCut(lowerCamelCase					,			stride=lowerCamelCase					,			name='shortcut'	)
							    if should_apply_shortcut
							    else tf.keras.layers.Activation('linear'					,			name='shortcut'	)
							)
							lowerCamelCase_  :    List[Any]   	=			ACTaFN[activation]





					def      __a    (		self    :    str					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
							lowerCamelCase_  :    int   	=			hidden_state
							lowerCamelCase_  :    Tuple   	=			self.conva(lowerCamelCase					,			training=lowerCamelCase	)
							lowerCamelCase_  :    List[Any]   	=			self.conva(lowerCamelCase					,			training=lowerCamelCase	)
							lowerCamelCase_  :    List[str]   	=			self.conva(lowerCamelCase					,			training=lowerCamelCase	)
							lowerCamelCase_  :    List[Any]   	=			self.shortcut(lowerCamelCase					,			training=lowerCamelCase	)
							hidden_state += residual
							lowerCamelCase_  :    int   	=			self.activation(lowerCamelCase	)
							return hidden_state






class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):



					def __init__(		self    :    int					,			lowerCamelCase    :    ResNetConfig					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int					,			lowerCamelCase    :    int = 2					,			lowerCamelCase    :    int = 2					,			**lowerCamelCase    :    List[Any]	):
							super().__init__(**lowerCamelCase	)

							lowerCamelCase_  :    List[Any]   	=			TFResNetBottleNeckLayer if config.layer_type == 'bottleneck' else TFResNetBasicLayer

							lowerCamelCase_  :    Tuple   	=			[layer(lowerCamelCase					,			lowerCamelCase					,			stride=lowerCamelCase					,			activation=config.hidden_act					,			name='layers.0'	)]
							layers += [
							    layer(lowerCamelCase					,			lowerCamelCase					,			activation=config.hidden_act					,			name=F"layers.{i + 1}"	)
							    for i in range(depth - 1	)
							]
							lowerCamelCase_  :    Dict   	=			layers





					def      __a    (		self    :    Any					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False	):
							for layer in self.stage_layers:
									lowerCamelCase_  :    Any   	=			layer(lowerCamelCase					,			training=lowerCamelCase	)
							return hidden_state






class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):



					def __init__(		self    :    List[Any]					,			lowerCamelCase    :    ResNetConfig					,			**lowerCamelCase    :    Optional[Any]	):
							super().__init__(**lowerCamelCase	)
							# based on `downsample_in_first_stage` the first layer of the first stage may or may not downsample the input
							lowerCamelCase_  :    List[Any]   	=			[
							    TFResNetStage(
							        lowerCamelCase					,			config.embedding_size					,			config.hidden_sizes[0]					,			stride=2 if config.downsample_in_first_stage else 1					,			depth=config.depths[0]					,			name='stages.0'					,			)
							]
							for i, (in_channels, out_channels, depth) in enumerate(
							    zip(config.hidden_sizes					,			config.hidden_sizes[1:]					,			config.depths[1:]	)	):
									self.stages.append(TFResNetStage(lowerCamelCase					,			lowerCamelCase					,			lowerCamelCase					,			depth=lowerCamelCase					,			name=F"stages.{i + 1}"	)	)





					def      __a    (		self    :    Optional[int]					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    bool = False					,			lowerCamelCase    :    bool = True					,			lowerCamelCase    :    bool = False					,			):
							lowerCamelCase_  :    Optional[int]   	=			() if output_hidden_states else None

							for stage_module in self.stages:
									if output_hidden_states:
											lowerCamelCase_  :    Any   	=			hidden_states + (hidden_state,)

									lowerCamelCase_  :    Optional[Any]   	=			stage_module(lowerCamelCase					,			training=lowerCamelCase	)

							if output_hidden_states:
									lowerCamelCase_  :    Union[str, Any]   	=			hidden_states + (hidden_state,)

							if not return_dict:
									return tuple(v for v in [hidden_state, hidden_states] if v is not None	)

							return TFBaseModelOutputWithNoAttention(last_hidden_state=lowerCamelCase					,			hidden_states=lowerCamelCase	)






class 	UpperCamelCase_     (     snake_case__       ):
					_a			:	int							=				ResNetConfig
					_a			:	Optional[Any]							=				'resnet'
					_a			:	int							=				'pixel_values'



					@property
					def      __a    (		self    :    Tuple	):
							return {"pixel_values": tf.TensorSpec(shape=(None, self.config.num_channels, 2_24, 2_24)					,			dtype=tf.floataa	)}
_lowercase		:     Union[str, Any] 					=r"""
    This model is a TensorFlow
    [tf.keras.layers.Layer](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) sub-class. Use it as a
    regular TensorFlow Module and refer to the TensorFlow documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`ResNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
"""


_lowercase		:     Optional[Any] 					=r"""
    Args:
        pixel_values (`tf.Tensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`ConvNextImageProcessor.__call__`] for details.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
"""






@keras_serializable
class 	UpperCamelCase_     (     tf.keras.layers.Layer       ):
					_a			:	int							=				ResNetConfig



					def __init__(		self    :    Dict					,			lowerCamelCase    :    ResNetConfig					,			**lowerCamelCase    :    List[str]	):
							super().__init__(**lowerCamelCase	)
							lowerCamelCase_  :    Tuple   	=			config
							lowerCamelCase_  :    int   	=			TFResNetEmbeddings(lowerCamelCase					,			name='embedder'	)
							lowerCamelCase_  :    Union[str, Any]   	=			TFResNetEncoder(lowerCamelCase					,			name='encoder'	)
							lowerCamelCase_  :    Tuple   	=			tf.keras.layers.GlobalAveragePoolingaD(keepdims=lowerCamelCase	)





					@unpack_inputs
					def      __a    (		self    :    List[str]					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    Optional[bool] = None					,			lowerCamelCase    :    Optional[bool] = None					,			lowerCamelCase    :    bool = False					,			):
							lowerCamelCase_  :    str   	=			(
							    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
							)
							lowerCamelCase_  :    Optional[Any]   	=			return_dict if return_dict is not None else self.config.use_return_dict

							# TF 2.0 image layers can't use NCHW format when running on CPU.
							# We transpose to NHWC format and then transpose back after the full forward pass.
							# (batch_size, num_channels, height, width) -> (batch_size, height, width, num_channels)
							lowerCamelCase_  :    str   	=			tf.transpose(lowerCamelCase					,			perm=[0, 2, 3, 1]	)
							lowerCamelCase_  :    Any   	=			self.embedder(lowerCamelCase					,			training=lowerCamelCase	)

							lowerCamelCase_  :    str   	=			self.encoder(
							    lowerCamelCase					,			output_hidden_states=lowerCamelCase					,			return_dict=lowerCamelCase					,			training=lowerCamelCase	)

							lowerCamelCase_  :    Optional[int]   	=			encoder_outputs[0]

							lowerCamelCase_  :    str   	=			self.pooler(lowerCamelCase	)

							# Transpose all the outputs to the NCHW format
							# (batch_size, height, width, num_channels) -> (batch_size, num_channels, height, width)
							lowerCamelCase_  :    str   	=			tf.transpose(lowerCamelCase					,			(0, 3, 1, 2)	)
							lowerCamelCase_  :    Tuple   	=			tf.transpose(lowerCamelCase					,			(0, 3, 1, 2)	)
							lowerCamelCase_  :    Dict   	=			()
							for hidden_state in encoder_outputs[1:]:
									lowerCamelCase_  :    Dict   	=			hidden_states + tuple(tf.transpose(lowerCamelCase					,			(0, 3, 1, 2)	) for h in hidden_state	)

							if not return_dict:
									return (last_hidden_state, pooled_output) + hidden_states

							lowerCamelCase_  :    Dict   	=			hidden_states if output_hidden_states else None

							return TFBaseModelOutputWithPoolingAndNoAttention(
							    last_hidden_state=lowerCamelCase					,			pooler_output=lowerCamelCase					,			hidden_states=lowerCamelCase					,			)






@add_start_docstrings(
    'The bare ResNet model outputting raw features without any specific head on top.'     ,    snake_case__     ,    )
class 	UpperCamelCase_     (     snake_case__       ):



					def __init__(		self    :    Any					,			lowerCamelCase    :    ResNetConfig					,			**lowerCamelCase    :    int	):
							super().__init__(lowerCamelCase					,			**lowerCamelCase	)
							lowerCamelCase_  :    List[str]   	=			TFResNetMainLayer(config=lowerCamelCase					,			name='resnet'	)





					@add_start_docstrings_to_model_forward(lowerCamelCase	)
					@add_code_sample_docstrings(
					    checkpoint=_CHECKPOINT_FOR_DOC					,			output_type=lowerCamelCase					,			config_class=_CONFIG_FOR_DOC					,			modality='vision'					,			expected_output=_EXPECTED_OUTPUT_SHAPE					,			)
					@unpack_inputs
					def      __a    (		self    :    List[str]					,			lowerCamelCase    :    tf.Tensor					,			lowerCamelCase    :    Optional[bool] = None					,			lowerCamelCase    :    Optional[bool] = None					,			lowerCamelCase    :    bool = False					,			):
							lowerCamelCase_  :    List[str]   	=			(
							    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
							)
							lowerCamelCase_  :    Tuple   	=			return_dict if return_dict is not None else self.config.use_return_dict

							lowerCamelCase_  :    Dict   	=			self.resnet(
							    pixel_values=lowerCamelCase					,			output_hidden_states=lowerCamelCase					,			return_dict=lowerCamelCase					,			training=lowerCamelCase					,			)
							return resnet_outputs






@add_start_docstrings(
    '\n    ResNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    '     ,    snake_case__     ,    )
class 	UpperCamelCase_     (     snake_case__     ,    snake_case__       ):



					def __init__(		self    :    Tuple					,			lowerCamelCase    :    ResNetConfig					,			**lowerCamelCase    :    Optional[Any]	):
							super().__init__(lowerCamelCase					,			**lowerCamelCase	)
							lowerCamelCase_  :    Tuple   	=			config.num_labels
							lowerCamelCase_  :    Union[str, Any]   	=			TFResNetMainLayer(lowerCamelCase					,			name='resnet'	)
							# classification head
							lowerCamelCase_  :    Dict   	=			(
							    tf.keras.layers.Dense(config.num_labels					,			name='classifier.1'	)
							    if config.num_labels > 0
							    else tf.keras.layers.Activation('linear'					,			name='classifier.1'	)
							)



					def      __a    (		self    :    List[str]					,			lowerCamelCase    :    tf.Tensor	):
							lowerCamelCase_  :    Tuple   	=			tf.keras.layers.Flatten()(lowerCamelCase	)
							lowerCamelCase_  :    Dict   	=			self.classifier_layer(lowerCamelCase	)
							return logits





					@add_start_docstrings_to_model_forward(lowerCamelCase	)
					@add_code_sample_docstrings(
					    checkpoint=_IMAGE_CLASS_CHECKPOINT					,			output_type=lowerCamelCase					,			config_class=_CONFIG_FOR_DOC					,			expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT					,			)
					@unpack_inputs
					def      __a    (		self    :    Optional[int]					,			lowerCamelCase    :    tf.Tensor = None					,			lowerCamelCase    :    tf.Tensor = None					,			lowerCamelCase    :    bool = None					,			lowerCamelCase    :    bool = None					,			lowerCamelCase    :    bool = False					,			):
							lowerCamelCase_  :    List[str]   	=			return_dict if return_dict is not None else self.config.use_return_dict

							lowerCamelCase_  :    Optional[int]   	=			self.resnet(
							    lowerCamelCase					,			output_hidden_states=lowerCamelCase					,			return_dict=lowerCamelCase					,			training=lowerCamelCase	)

							lowerCamelCase_  :    Union[str, Any]   	=			outputs.pooler_output if return_dict else outputs[1]

							lowerCamelCase_  :    Union[str, Any]   	=			self.classifier(lowerCamelCase	)

							lowerCamelCase_  :    List[Any]   	=			None if labels is None else self.hf_compute_loss(lowerCamelCase					,			lowerCamelCase	)

							if not return_dict:
									lowerCamelCase_  :    int   	=			(logits,) + outputs[2:]
									return (loss,) + output if loss is not None else output

							return TFImageClassifierOutputWithNoAttention(loss=lowerCamelCase					,			logits=lowerCamelCase					,			hidden_states=outputs.hidden_states	)
