


import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    RemBertConfig,
							    RemBertForCausalLM,
							    RemBertForMaskedLM,
							    RemBertForMultipleChoice,
							    RemBertForQuestionAnswering,
							    RemBertForSequenceClassification,
							    RemBertForTokenClassification,
							    RemBertModel,
							)
							from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST



class    __lowercase  :

							"""simple docstring"""


							def __init__(    self				, A_				, A_=13				, A_=7				, A_=True				, A_=True				, A_=True				, A_=True				, A_=99				, A_=32				, A_=18				, A_=43				, A_=5				, A_=4				, A_=37				, A_="gelu"				, A_=0.1				, A_=0.1				, A_=512				, A_=16				, A_=2				, A_=0.02				, A_=3				, A_=4				, A_=None				, )->	Any:
											_SCREAMING_SNAKE_CASE      = parent
											_SCREAMING_SNAKE_CASE      = batch_size
											_SCREAMING_SNAKE_CASE      = seq_length
											_SCREAMING_SNAKE_CASE      = is_training
											_SCREAMING_SNAKE_CASE      = use_input_mask
											_SCREAMING_SNAKE_CASE      = use_token_type_ids
											_SCREAMING_SNAKE_CASE      = use_labels
											_SCREAMING_SNAKE_CASE      = vocab_size
											_SCREAMING_SNAKE_CASE      = hidden_size
											_SCREAMING_SNAKE_CASE      = input_embedding_size
											_SCREAMING_SNAKE_CASE      = output_embedding_size
											_SCREAMING_SNAKE_CASE      = num_hidden_layers
											_SCREAMING_SNAKE_CASE      = num_attention_heads
											_SCREAMING_SNAKE_CASE      = intermediate_size
											_SCREAMING_SNAKE_CASE      = hidden_act
											_SCREAMING_SNAKE_CASE      = hidden_dropout_prob
											_SCREAMING_SNAKE_CASE      = attention_probs_dropout_prob
											_SCREAMING_SNAKE_CASE      = max_position_embeddings
											_SCREAMING_SNAKE_CASE      = type_vocab_size
											_SCREAMING_SNAKE_CASE      = type_sequence_label_size
											_SCREAMING_SNAKE_CASE      = initializer_range
											_SCREAMING_SNAKE_CASE      = num_labels
											_SCREAMING_SNAKE_CASE      = num_choices
											_SCREAMING_SNAKE_CASE      = scope





							def 		__magic_name__			(    self	)->	Union[str, Any]:
											_SCREAMING_SNAKE_CASE      = ids_tensor([self.batch_size, self.seq_length]				, self.vocab_size	)

											_SCREAMING_SNAKE_CASE      = None
											if self.use_input_mask:
															_SCREAMING_SNAKE_CASE      = random_attention_mask([self.batch_size, self.seq_length]	)

											_SCREAMING_SNAKE_CASE      = None
											if self.use_token_type_ids:
															_SCREAMING_SNAKE_CASE      = ids_tensor([self.batch_size, self.seq_length]				, self.type_vocab_size	)

											_SCREAMING_SNAKE_CASE      = None
											_SCREAMING_SNAKE_CASE      = None
											_SCREAMING_SNAKE_CASE      = None
											if self.use_labels:
															_SCREAMING_SNAKE_CASE      = ids_tensor([self.batch_size]				, self.type_sequence_label_size	)
															_SCREAMING_SNAKE_CASE      = ids_tensor([self.batch_size, self.seq_length]				, self.num_labels	)
															_SCREAMING_SNAKE_CASE      = ids_tensor([self.batch_size]				, self.num_choices	)

											_SCREAMING_SNAKE_CASE      = RemBertConfig(
											    vocab_size=self.vocab_size				, hidden_size=self.hidden_size				, input_embedding_size=self.input_embedding_size				, output_embedding_size=self.output_embedding_size				, num_hidden_layers=self.num_hidden_layers				, num_attention_heads=self.num_attention_heads				, intermediate_size=self.intermediate_size				, hidden_act=self.hidden_act				, hidden_dropout_prob=self.hidden_dropout_prob				, attention_probs_dropout_prob=self.attention_probs_dropout_prob				, max_position_embeddings=self.max_position_embeddings				, type_vocab_size=self.type_vocab_size				, is_decoder=A_				, initializer_range=self.initializer_range				, )

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





							def 		__magic_name__			(    self	)->	List[Any]:
											(
											    (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      
											)      = self.prepare_config_and_inputs()

											_SCREAMING_SNAKE_CASE      = True
											_SCREAMING_SNAKE_CASE      = floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
											_SCREAMING_SNAKE_CASE      = ids_tensor([self.batch_size, self.seq_length]				, vocab_size=2	)

											return (
											    config,
											    input_ids,
											    token_type_ids,
											    input_mask,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)





							def 		__magic_name__			(    self				, A_				, A_				, A_				, A_				, A_				, A_				, A_	)->	Optional[int]:
											_SCREAMING_SNAKE_CASE      = RemBertModel(config=A_	)
											model.to(A_	)
											model.eval()
											_SCREAMING_SNAKE_CASE      = model(A_				, attention_mask=A_				, token_type_ids=A_	)
											_SCREAMING_SNAKE_CASE      = model(A_				, token_type_ids=A_	)
											_SCREAMING_SNAKE_CASE      = model(A_	)
											self.parent.assertEqual(result.last_hidden_state.shape				, (self.batch_size, self.seq_length, self.hidden_size)	)





							def 		__magic_name__			(    self				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, )->	int:
											_SCREAMING_SNAKE_CASE      = True
											_SCREAMING_SNAKE_CASE      = RemBertModel(A_	)
											model.to(A_	)
											model.eval()
											_SCREAMING_SNAKE_CASE      = model(
											    A_				, attention_mask=A_				, token_type_ids=A_				, encoder_hidden_states=A_				, encoder_attention_mask=A_				, )
											_SCREAMING_SNAKE_CASE      = model(
											    A_				, attention_mask=A_				, token_type_ids=A_				, encoder_hidden_states=A_				, )
											_SCREAMING_SNAKE_CASE      = model(A_				, attention_mask=A_				, token_type_ids=A_	)
											self.parent.assertEqual(result.last_hidden_state.shape				, (self.batch_size, self.seq_length, self.hidden_size)	)





							def 		__magic_name__			(    self				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, )->	Any:
											_SCREAMING_SNAKE_CASE      = RemBertForCausalLM(config=A_	)
											model.to(A_	)
											model.eval()
											_SCREAMING_SNAKE_CASE      = model(A_				, attention_mask=A_				, token_type_ids=A_				, labels=A_	)
											self.parent.assertEqual(result.logits.shape				, (self.batch_size, self.seq_length, self.vocab_size)	)





							def 		__magic_name__			(    self				, A_				, A_				, A_				, A_				, A_				, A_				, A_	)->	Union[str, Any]:
											_SCREAMING_SNAKE_CASE      = RemBertForMaskedLM(config=A_	)
											model.to(A_	)
											model.eval()
											_SCREAMING_SNAKE_CASE      = model(A_				, attention_mask=A_				, token_type_ids=A_				, labels=A_	)
											self.parent.assertEqual(result.logits.shape				, (self.batch_size, self.seq_length, self.vocab_size)	)





							def 		__magic_name__			(    self				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, )->	Any:
											_SCREAMING_SNAKE_CASE      = True
											_SCREAMING_SNAKE_CASE      = True
											_SCREAMING_SNAKE_CASE      = RemBertForCausalLM(config=A_	)
											model.to(A_	)
											model.eval()

											# first forward pass
											_SCREAMING_SNAKE_CASE      = model(
											    A_				, attention_mask=A_				, encoder_hidden_states=A_				, encoder_attention_mask=A_				, use_cache=A_				, )
											_SCREAMING_SNAKE_CASE      = outputs.past_key_values

											# create hypothetical multiple next token and extent to next_input_ids
											_SCREAMING_SNAKE_CASE      = ids_tensor((self.batch_size, 3)				, config.vocab_size	)
											_SCREAMING_SNAKE_CASE      = ids_tensor((self.batch_size, 3)				, vocab_size=2	)

											# append to next input_ids and
											_SCREAMING_SNAKE_CASE      = torch.cat([input_ids, next_tokens]				, dim=-1	)
											_SCREAMING_SNAKE_CASE      = torch.cat([input_mask, next_mask]				, dim=-1	)

											_SCREAMING_SNAKE_CASE      = model(
											    A_				, attention_mask=A_				, encoder_hidden_states=A_				, encoder_attention_mask=A_				, output_hidden_states=A_				, )['hidden_states'][0]
											_SCREAMING_SNAKE_CASE      = model(
											    A_				, attention_mask=A_				, encoder_hidden_states=A_				, encoder_attention_mask=A_				, past_key_values=A_				, output_hidden_states=A_				, )['hidden_states'][0]

											# select random slice
											_SCREAMING_SNAKE_CASE      = ids_tensor((1,)				, output_from_past.shape[-1]	).item()
											_SCREAMING_SNAKE_CASE      = output_from_no_past[:, -3:, random_slice_idx].detach()
											_SCREAMING_SNAKE_CASE      = output_from_past[:, :, random_slice_idx].detach()

											self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

											# test that outputs are equal for slice
											self.parent.assertTrue(torch.allclose(A_				, A_				, atol=1e-3	)	)





							def 		__magic_name__			(    self				, A_				, A_				, A_				, A_				, A_				, A_				, A_	)->	Optional[int]:
											_SCREAMING_SNAKE_CASE      = RemBertForQuestionAnswering(config=A_	)
											model.to(A_	)
											model.eval()
											_SCREAMING_SNAKE_CASE      = model(
											    A_				, attention_mask=A_				, token_type_ids=A_				, start_positions=A_				, end_positions=A_				, )
											self.parent.assertEqual(result.start_logits.shape				, (self.batch_size, self.seq_length)	)
											self.parent.assertEqual(result.end_logits.shape				, (self.batch_size, self.seq_length)	)





							def 		__magic_name__			(    self				, A_				, A_				, A_				, A_				, A_				, A_				, A_	)->	Any:
											_SCREAMING_SNAKE_CASE      = self.num_labels
											_SCREAMING_SNAKE_CASE      = RemBertForSequenceClassification(A_	)
											model.to(A_	)
											model.eval()
											_SCREAMING_SNAKE_CASE      = model(A_				, attention_mask=A_				, token_type_ids=A_				, labels=A_	)
											self.parent.assertEqual(result.logits.shape				, (self.batch_size, self.num_labels)	)





							def 		__magic_name__			(    self				, A_				, A_				, A_				, A_				, A_				, A_				, A_	)->	Optional[Any]:
											_SCREAMING_SNAKE_CASE      = self.num_labels
											_SCREAMING_SNAKE_CASE      = RemBertForTokenClassification(config=A_	)
											model.to(A_	)
											model.eval()
											_SCREAMING_SNAKE_CASE      = model(A_				, attention_mask=A_				, token_type_ids=A_				, labels=A_	)
											self.parent.assertEqual(result.logits.shape				, (self.batch_size, self.seq_length, self.num_labels)	)





							def 		__magic_name__			(    self				, A_				, A_				, A_				, A_				, A_				, A_				, A_	)->	Any:
											_SCREAMING_SNAKE_CASE      = self.num_choices
											_SCREAMING_SNAKE_CASE      = RemBertForMultipleChoice(config=A_	)
											model.to(A_	)
											model.eval()
											_SCREAMING_SNAKE_CASE      = input_ids.unsqueeze(1	).expand(-1				, self.num_choices				, -1	).contiguous()
											_SCREAMING_SNAKE_CASE      = token_type_ids.unsqueeze(1	).expand(-1				, self.num_choices				, -1	).contiguous()
											_SCREAMING_SNAKE_CASE      = input_mask.unsqueeze(1	).expand(-1				, self.num_choices				, -1	).contiguous()
											_SCREAMING_SNAKE_CASE      = model(
											    A_				, attention_mask=A_				, token_type_ids=A_				, labels=A_				, )
											self.parent.assertEqual(result.logits.shape				, (self.batch_size, self.num_choices)	)





							def 		__magic_name__			(    self	)->	Union[str, Any]:
											_SCREAMING_SNAKE_CASE      = self.prepare_config_and_inputs()
											(
											    (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      
											)      = config_and_inputs
											_SCREAMING_SNAKE_CASE      = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
											return config, inputs_dict







@require_torch
class    __lowercase  (       UpperCamelCase      ,     UpperCamelCase      ,     unittest.TestCase							):

							"""simple docstring"""


							SCREAMING_SNAKE_CASE   :					List[Any]        =						(
							    (
							        RemBertModel,
							        RemBertForMaskedLM,
							        RemBertForCausalLM,
							        RemBertForMultipleChoice,
							        RemBertForQuestionAnswering,
							        RemBertForSequenceClassification,
							        RemBertForTokenClassification,
							    )
							    if is_torch_available()
							    else ()
							)
							SCREAMING_SNAKE_CASE   :					List[Any]        =						(RemBertForCausalLM,) if is_torch_available() else ()
							SCREAMING_SNAKE_CASE   :					int        =						(
							    {
							        "feature-extraction": RemBertModel,
							        "fill-mask": RemBertForMaskedLM,
							        "question-answering": RemBertForQuestionAnswering,
							        "text-classification": RemBertForSequenceClassification,
							        "text-generation": RemBertForCausalLM,
							        "token-classification": RemBertForTokenClassification,
							        "zero-shot": RemBertForSequenceClassification,
							    }
							    if is_torch_available()
							    else {}
							)





							def 		__magic_name__			(    self	)->	List[str]:
											_SCREAMING_SNAKE_CASE      = RemBertModelTester(self	)
											_SCREAMING_SNAKE_CASE      = ConfigTester(self				, config_class=A_				, hidden_size=37	)





							def 		__magic_name__			(    self	)->	Optional[int]:
											self.config_tester.run_common_tests()





							def 		__magic_name__			(    self	)->	Optional[Any]:
											_SCREAMING_SNAKE_CASE      = self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*A_	)





							def 		__magic_name__			(    self	)->	Union[str, Any]:
											_SCREAMING_SNAKE_CASE      = self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
															_SCREAMING_SNAKE_CASE      = type
															self.model_tester.create_and_check_model(*A_	)





							def 		__magic_name__			(    self	)->	Any:
											_SCREAMING_SNAKE_CASE      = self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*A_	)





							def 		__magic_name__			(    self	)->	Any:
											_SCREAMING_SNAKE_CASE      = self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*A_	)





							def 		__magic_name__			(    self	)->	Optional[int]:
											_SCREAMING_SNAKE_CASE      = self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_decoder_model_past_large_inputs(*A_	)





							def 		__magic_name__			(    self	)->	Tuple:
											_SCREAMING_SNAKE_CASE      = self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*A_	)





							def 		__magic_name__			(    self	)->	Tuple:
											_SCREAMING_SNAKE_CASE      = self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*A_	)





							def 		__magic_name__			(    self	)->	List[Any]:
											_SCREAMING_SNAKE_CASE      = self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*A_	)





							def 		__magic_name__			(    self	)->	Union[str, Any]:
											_SCREAMING_SNAKE_CASE      = self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_model_as_decoder(*A_	)





							def 		__magic_name__			(    self	)->	str:
											# This regression test was failing with PyTorch < 1.3
											(
											    (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      (
											    _SCREAMING_SNAKE_CASE
											)      ,      
											)      = self.model_tester.prepare_config_and_inputs_for_decoder()

											_SCREAMING_SNAKE_CASE      = None

											self.model_tester.create_and_check_model_as_decoder(
											    A_				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, A_				, )





							@slow
							def 		__magic_name__			(    self	)->	Optional[int]:
											for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															_SCREAMING_SNAKE_CASE      = RemBertModel.from_pretrained(A_	)
															self.assertIsNotNone(A_	)







@require_torch
class    __lowercase  (       unittest.TestCase							):

							"""simple docstring"""


							@slow
							def 		__magic_name__			(    self	)->	str:
											# Test exact values at the last hidden layer
											_SCREAMING_SNAKE_CASE      = RemBertModel.from_pretrained('google/rembert'	)
											_SCREAMING_SNAKE_CASE      = torch.tensor([[312, 56498, 313, 2125, 313]]	)
											_SCREAMING_SNAKE_CASE      = torch.tensor([[0, 0, 0, 1, 1]]	)
											with torch.no_grad():
															_SCREAMING_SNAKE_CASE      = model(A_				, token_type_ids=A_				, output_hidden_states=A_	)

											_SCREAMING_SNAKE_CASE      = 1152

											_SCREAMING_SNAKE_CASE      = torch.Size((1, 5, hidden_size)	)
											self.assertEqual(output['last_hidden_state'].shape				, A_	)

											_SCREAMING_SNAKE_CASE      = torch.tensor(
											    [
											        [
											            [0.0754, -0.2022, 0.1904],
											            [-0.3354, -0.3692, -0.4791],
											            [-0.2314, -0.6729, -0.0749],
											            [-0.0396, -0.3105, -0.4234],
											            [-0.1571, -0.0525, 0.5353],
											        ]
											    ]	)

											# Running on the original tf implementation gives slightly different results here.
											# Not clear why this variations is present
											# TODO: Find reason for discrepancy
											# expected_original_implementation = [[
											#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
											#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
											#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
											#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
											#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
											# ]]

											self.assertTrue(torch.allclose(output['last_hidden_state'][:, :, :3]				, A_				, atol=1e-4	)	)




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


snake_case			:      Optional[int] 							=   {'configuration_vit_hybrid': ['VIT_HYBRID_PRETRAINED_CONFIG_ARCHIVE_MAP', 'ViTHybridConfig']}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							snake_case			:      int 							=   [
							    'VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'ViTHybridForImageClassification',
							    'ViTHybridModel',
							    'ViTHybridPreTrainedModel',
							]

try:
							if not is_vision_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							snake_case			:      Any 							=   ['ViTHybridImageProcessor']


if TYPE_CHECKING:
							from .configuration_vit_hybrid import VIT_HYBRID_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTHybridConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_vit_hybrid import (
														    VIT_HYBRID_PRETRAINED_MODEL_ARCHIVE_LIST,
														    ViTHybridForImageClassification,
														    ViTHybridModel,
														    ViTHybridPreTrainedModel,
														)

							try:
														if not is_vision_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .image_processing_vit_hybrid import ViTHybridImageProcessor


else:
							import sys

							snake_case			:      Optional[int] 							=   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
