




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    RemBertConfig,
					    RemBertForCausalLM,
					    RemBertForMaskedLM,
					    RemBertForMultipleChoice,
					    RemBertForQuestionAnswering,
					    RemBertForSequenceClassification,
					    RemBertForTokenClassification,
					    RemBertModel,
					)
					from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class      a__       :






						def __init__(  self						,     _A						,     _A=1_3						,     _A=7						,     _A=True						,     _A=True						,     _A=True						,     _A=True						,     _A=9_9						,     _A=3_2						,     _A=1_8						,     _A=4_3						,     _A=5						,     _A=4						,     _A=3_7						,     _A="gelu"						,     _A=0.1						,     _A=0.1						,     _A=5_1_2						,     _A=1_6						,     _A=2						,     _A=0.02						,     _A=3						,     _A=4						,     _A=None						,     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			parent
											__lowerCAmelCase							  =			batch_size
											__lowerCAmelCase							  =			seq_length
											__lowerCAmelCase							  =			is_training
											__lowerCAmelCase							  =			use_input_mask
											__lowerCAmelCase							  =			use_token_type_ids
											__lowerCAmelCase							  =			use_labels
											__lowerCAmelCase							  =			vocab_size
											__lowerCAmelCase							  =			hidden_size
											__lowerCAmelCase							  =			input_embedding_size
											__lowerCAmelCase							  =			output_embedding_size
											__lowerCAmelCase							  =			num_hidden_layers
											__lowerCAmelCase							  =			num_attention_heads
											__lowerCAmelCase							  =			intermediate_size
											__lowerCAmelCase							  =			hidden_act
											__lowerCAmelCase							  =			hidden_dropout_prob
											__lowerCAmelCase							  =			attention_probs_dropout_prob
											__lowerCAmelCase							  =			max_position_embeddings
											__lowerCAmelCase							  =			type_vocab_size
											__lowerCAmelCase							  =			type_sequence_label_size
											__lowerCAmelCase							  =			initializer_range
											__lowerCAmelCase							  =			num_labels
											__lowerCAmelCase							  =			num_choices
											__lowerCAmelCase							  =			scope






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			ids_tensor([self.batch_size, self.seq_length]						,     self.vocab_size     )

											__lowerCAmelCase							  =			None
											if self.use_input_mask:
																__lowerCAmelCase							  =			random_attention_mask([self.batch_size, self.seq_length]     )

											__lowerCAmelCase							  =			None
											if self.use_token_type_ids:
																__lowerCAmelCase							  =			ids_tensor([self.batch_size, self.seq_length]						,     self.type_vocab_size     )

											__lowerCAmelCase							  =			None
											__lowerCAmelCase							  =			None
											__lowerCAmelCase							  =			None
											if self.use_labels:
																__lowerCAmelCase							  =			ids_tensor([self.batch_size]						,     self.type_sequence_label_size     )
																__lowerCAmelCase							  =			ids_tensor([self.batch_size, self.seq_length]						,     self.num_labels     )
																__lowerCAmelCase							  =			ids_tensor([self.batch_size]						,     self.num_choices     )

											__lowerCAmelCase							  =			RemBertConfig(
											    vocab_size=self.vocab_size						,     hidden_size=self.hidden_size						,     input_embedding_size=self.input_embedding_size						,     output_embedding_size=self.output_embedding_size						,     num_hidden_layers=self.num_hidden_layers						,     num_attention_heads=self.num_attention_heads						,     intermediate_size=self.intermediate_size						,     hidden_act=self.hidden_act						,     hidden_dropout_prob=self.hidden_dropout_prob						,     attention_probs_dropout_prob=self.attention_probs_dropout_prob						,     max_position_embeddings=self.max_position_embeddings						,     type_vocab_size=self.type_vocab_size						,     is_decoder=_A						,     initializer_range=self.initializer_range						,     )

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											(
											    (
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				
											)							  =			self.prepare_config_and_inputs()

											__lowerCAmelCase							  =			True
											__lowerCAmelCase							  =			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]     )
											__lowerCAmelCase							  =			ids_tensor([self.batch_size, self.seq_length]						,     vocab_size=2     )

											return (
											    config,
											    input_ids,
											    token_type_ids,
											    input_mask,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			RemBertModel(config=_A     )
											model.to(_A     )
											model.eval()
											__lowerCAmelCase							  =			model(_A						,     attention_mask=_A						,     token_type_ids=_A     )
											__lowerCAmelCase							  =			model(_A						,     token_type_ids=_A     )
											__lowerCAmelCase							  =			model(_A     )
											self.parent.assertEqual(result.last_hidden_state.shape						,     (self.batch_size, self.seq_length, self.hidden_size)     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			True
											__lowerCAmelCase							  =			RemBertModel(_A     )
											model.to(_A     )
											model.eval()
											__lowerCAmelCase							  =			model(
											    _A						,     attention_mask=_A						,     token_type_ids=_A						,     encoder_hidden_states=_A						,     encoder_attention_mask=_A						,     )
											__lowerCAmelCase							  =			model(
											    _A						,     attention_mask=_A						,     token_type_ids=_A						,     encoder_hidden_states=_A						,     )
											__lowerCAmelCase							  =			model(_A						,     attention_mask=_A						,     token_type_ids=_A     )
											self.parent.assertEqual(result.last_hidden_state.shape						,     (self.batch_size, self.seq_length, self.hidden_size)     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			RemBertForCausalLM(config=_A     )
											model.to(_A     )
											model.eval()
											__lowerCAmelCase							  =			model(_A						,     attention_mask=_A						,     token_type_ids=_A						,     labels=_A     )
											self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.seq_length, self.vocab_size)     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			RemBertForMaskedLM(config=_A     )
											model.to(_A     )
											model.eval()
											__lowerCAmelCase							  =			model(_A						,     attention_mask=_A						,     token_type_ids=_A						,     labels=_A     )
											self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.seq_length, self.vocab_size)     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			True
											__lowerCAmelCase							  =			True
											__lowerCAmelCase							  =			RemBertForCausalLM(config=_A     )
											model.to(_A     )
											model.eval()

											# first forward pass
											__lowerCAmelCase							  =			model(
											    _A						,     attention_mask=_A						,     encoder_hidden_states=_A						,     encoder_attention_mask=_A						,     use_cache=_A						,     )
											__lowerCAmelCase							  =			outputs.past_key_values

											# create hypothetical multiple next token and extent to next_input_ids
											__lowerCAmelCase							  =			ids_tensor((self.batch_size, 3)						,     config.vocab_size     )
											__lowerCAmelCase							  =			ids_tensor((self.batch_size, 3)						,     vocab_size=2     )

											# append to next input_ids and
											__lowerCAmelCase							  =			torch.cat([input_ids, next_tokens]						,     dim=-1     )
											__lowerCAmelCase							  =			torch.cat([input_mask, next_mask]						,     dim=-1     )

											__lowerCAmelCase							  =			model(
											    _A						,     attention_mask=_A						,     encoder_hidden_states=_A						,     encoder_attention_mask=_A						,     output_hidden_states=_A						,     )["hidden_states"][0]
											__lowerCAmelCase							  =			model(
											    _A						,     attention_mask=_A						,     encoder_hidden_states=_A						,     encoder_attention_mask=_A						,     past_key_values=_A						,     output_hidden_states=_A						,     )["hidden_states"][0]

											# select random slice
											__lowerCAmelCase							  =			ids_tensor((1,)						,     output_from_past.shape[-1]     ).item()
											__lowerCAmelCase							  =			output_from_no_past[:, -3:, random_slice_idx].detach()
											__lowerCAmelCase							  =			output_from_past[:, :, random_slice_idx].detach()

											self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

											# test that outputs are equal for slice
											self.parent.assertTrue(torch.allclose(_A						,     _A						,     atol=1E-3     )     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			RemBertForQuestionAnswering(config=_A     )
											model.to(_A     )
											model.eval()
											__lowerCAmelCase							  =			model(
											    _A						,     attention_mask=_A						,     token_type_ids=_A						,     start_positions=_A						,     end_positions=_A						,     )
											self.parent.assertEqual(result.start_logits.shape						,     (self.batch_size, self.seq_length)     )
											self.parent.assertEqual(result.end_logits.shape						,     (self.batch_size, self.seq_length)     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.num_labels
											__lowerCAmelCase							  =			RemBertForSequenceClassification(_A     )
											model.to(_A     )
											model.eval()
											__lowerCAmelCase							  =			model(_A						,     attention_mask=_A						,     token_type_ids=_A						,     labels=_A     )
											self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.num_labels)     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.num_labels
											__lowerCAmelCase							  =			RemBertForTokenClassification(config=_A     )
											model.to(_A     )
											model.eval()
											__lowerCAmelCase							  =			model(_A						,     attention_mask=_A						,     token_type_ids=_A						,     labels=_A     )
											self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.seq_length, self.num_labels)     )






						def 							__SCREAMING_SNAKE_CASE(  self						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.num_choices
											__lowerCAmelCase							  =			RemBertForMultipleChoice(config=_A     )
											model.to(_A     )
											model.eval()
											__lowerCAmelCase							  =			input_ids.unsqueeze(1     ).expand(-1						,     self.num_choices						,     -1     ).contiguous()
											__lowerCAmelCase							  =			token_type_ids.unsqueeze(1     ).expand(-1						,     self.num_choices						,     -1     ).contiguous()
											__lowerCAmelCase							  =			input_mask.unsqueeze(1     ).expand(-1						,     self.num_choices						,     -1     ).contiguous()
											__lowerCAmelCase							  =			model(
											    _A						,     attention_mask=_A						,     token_type_ids=_A						,     labels=_A						,     )
											self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.num_choices)     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.prepare_config_and_inputs()
											(
											    (
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				
											)							  =			config_and_inputs
											__lowerCAmelCase							  =			{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
											return config, inputs_dict






@require_torch
class      a__       (   snake_case__					,							snake_case__					,							unittest.TestCase ):
						_a				:       Optional[Any]   =       (
						    (
						        RemBertModel,
						        RemBertForMaskedLM,
						        RemBertForCausalLM,
						        RemBertForMultipleChoice,
						        RemBertForQuestionAnswering,
						        RemBertForSequenceClassification,
						        RemBertForTokenClassification,
						    )
						    if is_torch_available()
						    else ()
						)
						_a				:       List[str]   =       (RemBertForCausalLM,) if is_torch_available() else ()
						_a				:       Any   =       (
						    {
						        """feature-extraction""": RemBertModel,
						        """fill-mask""": RemBertForMaskedLM,
						        """question-answering""": RemBertForQuestionAnswering,
						        """text-classification""": RemBertForSequenceClassification,
						        """text-generation""": RemBertForCausalLM,
						        """token-classification""": RemBertForTokenClassification,
						        """zero-shot""": RemBertForSequenceClassification,
						    }
						    if is_torch_available()
						    else {}
						)






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			RemBertModelTester(self     )
											__lowerCAmelCase							  =			ConfigTester(self						,     config_class=_A						,     hidden_size=3_7     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											self.config_tester.run_common_tests()






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*_A     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
																__lowerCAmelCase							  =			type
																self.model_tester.create_and_check_model(*_A     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*_A     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*_A     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_decoder_model_past_large_inputs(*_A     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*_A     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*_A     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*_A     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_model_as_decoder(*_A     )






						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											(
											    (
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				(
											    __lowerCAmelCase
											)  ,				
											)							  =			self.model_tester.prepare_config_and_inputs_for_decoder()

											__lowerCAmelCase							  =			None

											self.model_tester.create_and_check_model_as_decoder(
											    _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     _A						,     )






						@slow
						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																__lowerCAmelCase							  =			RemBertModel.from_pretrained(_A     )
																self.assertIsNotNone(_A     )






@require_torch
class      a__       (   unittest.TestCase ):






						@slow
						def 							__SCREAMING_SNAKE_CASE(  self     ):




											"""simple docstring"""



											__lowerCAmelCase							  =			RemBertModel.from_pretrained("google/rembert"     )
											__lowerCAmelCase							  =			torch.tensor([[3_1_2, 5_6_4_9_8, 3_1_3, 2_1_2_5, 3_1_3]]     )
											__lowerCAmelCase							  =			torch.tensor([[0, 0, 0, 1, 1]]     )
											with torch.no_grad():
																__lowerCAmelCase							  =			model(_A						,     token_type_ids=_A						,     output_hidden_states=_A     )

											__lowerCAmelCase							  =			1_1_5_2

											__lowerCAmelCase							  =			torch.Size((1, 5, hidden_size)     )
											self.assertEqual(output["last_hidden_state"].shape						,     _A     )

											__lowerCAmelCase							  =			torch.tensor(
											    [
											        [
											            [0.07_54, -0.20_22, 0.19_04],
											            [-0.33_54, -0.36_92, -0.47_91],
											            [-0.23_14, -0.67_29, -0.07_49],
											            [-0.03_96, -0.31_05, -0.42_34],
											            [-0.15_71, -0.05_25, 0.53_53],
											        ]
											    ]     )

											# Running on the original tf implementation gives slightly different results here.
											# Not clear why this variations is present
											# TODO: Find reason for discrepancy
											# expected_original_implementation = [[
											#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
											#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
											#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
											#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
											#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
											# ]]

											self.assertTrue(torch.allclose(output["last_hidden_state"][:, :, :3]						,     _A						,     atol=1E-4     )     )










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__            =     {
    """configuration_marian""": ["""MARIAN_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MarianConfig""", """MarianOnnxConfig"""],
}

try:
					if not is_sentencepiece_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__            =     ["""MarianTokenizer"""]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__            =     [
					    """MARIAN_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """MarianForCausalLM""",
					    """MarianModel""",
					    """MarianMTModel""",
					    """MarianPreTrainedModel""",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__            =     ["""TFMarianModel""", """TFMarianMTModel""", """TFMarianPreTrainedModel"""]

try:
					if not is_flax_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase__            =     ["""FlaxMarianModel""", """FlaxMarianMTModel""", """FlaxMarianPreTrainedModel"""]

if TYPE_CHECKING:
					from .configuration_marian import MARIAN_PRETRAINED_CONFIG_ARCHIVE_MAP, MarianConfig, MarianOnnxConfig

					try:
										if not is_sentencepiece_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_marian import MarianTokenizer

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_marian import (
										    MARIAN_PRETRAINED_MODEL_ARCHIVE_LIST,
										    MarianForCausalLM,
										    MarianModel,
										    MarianMTModel,
										    MarianPreTrainedModel,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_marian import TFMarianModel, TFMarianMTModel, TFMarianPreTrainedModel

					try:
										if not is_flax_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_flax_marian import FlaxMarianModel, FlaxMarianMTModel, FlaxMarianPreTrainedModel

else:
					import sys

					UpperCamelCase__            =     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




