




"""simple docstring"""





import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    RemBertConfig,
							    RemBertForCausalLM,
							    RemBertForMaskedLM,
							    RemBertForMultipleChoice,
							    RemBertForQuestionAnswering,
							    RemBertForSequenceClassification,
							    RemBertForTokenClassification,
							    RemBertModel,
							)
							from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST

class       __lowercase       :


				'''simple docstring'''


				def __init__(   self   :     Dict      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]=1_3      ,SCREAMING_SNAKE_CASE_   :     List[str]=7      ,SCREAMING_SNAKE_CASE_   :     Any=True      ,SCREAMING_SNAKE_CASE_   :     Optional[int]=True      ,SCREAMING_SNAKE_CASE_   :     Any=True      ,SCREAMING_SNAKE_CASE_   :     Dict=True      ,SCREAMING_SNAKE_CASE_   :     Any=9_9      ,SCREAMING_SNAKE_CASE_   :     List[str]=3_2      ,SCREAMING_SNAKE_CASE_   :     int=1_8      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]=4_3      ,SCREAMING_SNAKE_CASE_   :     Tuple=5      ,SCREAMING_SNAKE_CASE_   :     List[Any]=4      ,SCREAMING_SNAKE_CASE_   :     Optional[Any]=3_7      ,SCREAMING_SNAKE_CASE_   :     Any="gelu"      ,SCREAMING_SNAKE_CASE_   :     Dict=0.1      ,SCREAMING_SNAKE_CASE_   :     Tuple=0.1      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]=5_1_2      ,SCREAMING_SNAKE_CASE_   :     int=1_6      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]=2      ,SCREAMING_SNAKE_CASE_   :     int=0.02      ,SCREAMING_SNAKE_CASE_   :     int=3      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]=4      ,SCREAMING_SNAKE_CASE_   :     Tuple=None      ,):
										snake_case__       :					List[str]										=					parent
										snake_case__       :					int										=					batch_size
										snake_case__       :					Dict										=					seq_length
										snake_case__       :					Dict										=					is_training
										snake_case__       :					int										=					use_input_mask
										snake_case__       :					List[str]										=					use_token_type_ids
										snake_case__       :					str										=					use_labels
										snake_case__       :					List[Any]										=					vocab_size
										snake_case__       :					Union[str, Any]										=					hidden_size
										snake_case__       :					List[Any]										=					input_embedding_size
										snake_case__       :					List[Any]										=					output_embedding_size
										snake_case__       :					List[Any]										=					num_hidden_layers
										snake_case__       :					List[str]										=					num_attention_heads
										snake_case__       :					Optional[int]										=					intermediate_size
										snake_case__       :					Dict										=					hidden_act
										snake_case__       :					Dict										=					hidden_dropout_prob
										snake_case__       :					Any										=					attention_probs_dropout_prob
										snake_case__       :					List[Any]										=					max_position_embeddings
										snake_case__       :					Tuple										=					type_vocab_size
										snake_case__       :					Dict										=					type_sequence_label_size
										snake_case__       :					List[Any]										=					initializer_range
										snake_case__       :					List[Any]										=					num_labels
										snake_case__       :					Dict										=					num_choices
										snake_case__       :					Any										=					scope

				def       _lowerCamelCase					(   self   :     Optional[int]      ):
										snake_case__       :					Dict										=					ids_tensor([self.batch_size, self.seq_length]      ,self.vocab_size      )

										snake_case__       :					Optional[Any]										=					None
										if self.use_input_mask:
																snake_case__       :					List[str]										=					random_attention_mask([self.batch_size, self.seq_length]      )

										snake_case__       :					Tuple										=					None
										if self.use_token_type_ids:
																snake_case__       :					Tuple										=					ids_tensor([self.batch_size, self.seq_length]      ,self.type_vocab_size      )

										snake_case__       :					Optional[int]										=					None
										snake_case__       :					str										=					None
										snake_case__       :					Any										=					None
										if self.use_labels:
																snake_case__       :					Optional[int]										=					ids_tensor([self.batch_size]      ,self.type_sequence_label_size      )
																snake_case__       :					Dict										=					ids_tensor([self.batch_size, self.seq_length]      ,self.num_labels      )
																snake_case__       :					Tuple										=					ids_tensor([self.batch_size]      ,self.num_choices      )

										snake_case__       :					int										=					RemBertConfig(
										    vocab_size=self.vocab_size      ,hidden_size=self.hidden_size      ,input_embedding_size=self.input_embedding_size      ,output_embedding_size=self.output_embedding_size      ,num_hidden_layers=self.num_hidden_layers      ,num_attention_heads=self.num_attention_heads      ,intermediate_size=self.intermediate_size      ,hidden_act=self.hidden_act      ,hidden_dropout_prob=self.hidden_dropout_prob      ,attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,max_position_embeddings=self.max_position_embeddings      ,type_vocab_size=self.type_vocab_size      ,is_decoder=SCREAMING_SNAKE_CASE_      ,initializer_range=self.initializer_range      ,)

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

				def       _lowerCamelCase					(   self   :     Union[str, Any]      ):
										(
										    (
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	
										)       :					Dict										=					self.prepare_config_and_inputs()

										snake_case__       :					List[str]										=					True
										snake_case__       :					str										=					floats_tensor([self.batch_size, self.seq_length, self.hidden_size]      )
										snake_case__       :					Tuple										=					ids_tensor([self.batch_size, self.seq_length]      ,vocab_size=2      )

										return (
										    config,
										    input_ids,
										    token_type_ids,
										    input_mask,
										    sequence_labels,
										    token_labels,
										    choice_labels,
										    encoder_hidden_states,
										    encoder_attention_mask,
										)

				def       _lowerCamelCase					(   self   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     Tuple      ,SCREAMING_SNAKE_CASE_   :     Any      ,SCREAMING_SNAKE_CASE_   :     Tuple      ,SCREAMING_SNAKE_CASE_   :     Tuple      ,SCREAMING_SNAKE_CASE_   :     Tuple      ,SCREAMING_SNAKE_CASE_   :     Optional[Any]      ):
										snake_case__       :					str										=					RemBertModel(config=SCREAMING_SNAKE_CASE_      )
										model.to(SCREAMING_SNAKE_CASE_      )
										model.eval()
										snake_case__       :					Optional[Any]										=					model(SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      )
										snake_case__       :					Dict										=					model(SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      )
										snake_case__       :					Union[str, Any]										=					model(SCREAMING_SNAKE_CASE_      )
										self.parent.assertEqual(result.last_hidden_state.shape      ,(self.batch_size, self.seq_length, self.hidden_size)      )

				def       _lowerCamelCase					(   self   :     Optional[Any]      ,SCREAMING_SNAKE_CASE_   :     Dict      ,SCREAMING_SNAKE_CASE_   :     int      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]      ,SCREAMING_SNAKE_CASE_   :     Dict      ,SCREAMING_SNAKE_CASE_   :     List[Any]      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     Tuple      ,SCREAMING_SNAKE_CASE_   :     Any      ,SCREAMING_SNAKE_CASE_   :     Any      ,):
										snake_case__       :					List[str]										=					True
										snake_case__       :					str										=					RemBertModel(SCREAMING_SNAKE_CASE_      )
										model.to(SCREAMING_SNAKE_CASE_      )
										model.eval()
										snake_case__       :					int										=					model(
										    SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      ,encoder_hidden_states=SCREAMING_SNAKE_CASE_      ,encoder_attention_mask=SCREAMING_SNAKE_CASE_      ,)
										snake_case__       :					Union[str, Any]										=					model(
										    SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      ,encoder_hidden_states=SCREAMING_SNAKE_CASE_      ,)
										snake_case__       :					List[Any]										=					model(SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      )
										self.parent.assertEqual(result.last_hidden_state.shape      ,(self.batch_size, self.seq_length, self.hidden_size)      )

				def       _lowerCamelCase					(   self   :     Any      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]      ,SCREAMING_SNAKE_CASE_   :     List[Any]      ,SCREAMING_SNAKE_CASE_   :     List[str]      ,SCREAMING_SNAKE_CASE_   :     List[Any]      ,SCREAMING_SNAKE_CASE_   :     List[Any]      ,SCREAMING_SNAKE_CASE_   :     str      ,SCREAMING_SNAKE_CASE_   :     Dict      ,):
										snake_case__       :					List[str]										=					RemBertForCausalLM(config=SCREAMING_SNAKE_CASE_      )
										model.to(SCREAMING_SNAKE_CASE_      )
										model.eval()
										snake_case__       :					List[str]										=					model(SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      ,labels=SCREAMING_SNAKE_CASE_      )
										self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.seq_length, self.vocab_size)      )

				def       _lowerCamelCase					(   self   :     List[Any]      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]      ,SCREAMING_SNAKE_CASE_   :     int      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     Tuple      ,SCREAMING_SNAKE_CASE_   :     Any      ,SCREAMING_SNAKE_CASE_   :     Optional[Any]      ):
										snake_case__       :					Tuple										=					RemBertForMaskedLM(config=SCREAMING_SNAKE_CASE_      )
										model.to(SCREAMING_SNAKE_CASE_      )
										model.eval()
										snake_case__       :					List[Any]										=					model(SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      ,labels=SCREAMING_SNAKE_CASE_      )
										self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.seq_length, self.vocab_size)      )

				def       _lowerCamelCase					(   self   :     Tuple      ,SCREAMING_SNAKE_CASE_   :     Dict      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     int      ,SCREAMING_SNAKE_CASE_   :     List[Any]      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     Any      ,SCREAMING_SNAKE_CASE_   :     Any      ,SCREAMING_SNAKE_CASE_   :     Tuple      ,SCREAMING_SNAKE_CASE_   :     Tuple      ,):
										snake_case__       :					Union[str, Any]										=					True
										snake_case__       :					str										=					True
										snake_case__       :					Tuple										=					RemBertForCausalLM(config=SCREAMING_SNAKE_CASE_      )
										model.to(SCREAMING_SNAKE_CASE_      )
										model.eval()

										# first forward pass
										snake_case__       :					Optional[int]										=					model(
										    SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,encoder_hidden_states=SCREAMING_SNAKE_CASE_      ,encoder_attention_mask=SCREAMING_SNAKE_CASE_      ,use_cache=SCREAMING_SNAKE_CASE_      ,)
										snake_case__       :					Dict										=					outputs.past_key_values

										# create hypothetical multiple next token and extent to next_input_ids
										snake_case__       :					Any										=					ids_tensor((self.batch_size, 3)      ,config.vocab_size      )
										snake_case__       :					Dict										=					ids_tensor((self.batch_size, 3)      ,vocab_size=2      )

										# append to next input_ids and
										snake_case__       :					Union[str, Any]										=					torch.cat([input_ids, next_tokens]      ,dim=-1      )
										snake_case__       :					Tuple										=					torch.cat([input_mask, next_mask]      ,dim=-1      )

										snake_case__       :					Dict										=					model(
										    SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,encoder_hidden_states=SCREAMING_SNAKE_CASE_      ,encoder_attention_mask=SCREAMING_SNAKE_CASE_      ,output_hidden_states=SCREAMING_SNAKE_CASE_      ,)['''hidden_states'''][0]
										snake_case__       :					List[Any]										=					model(
										    SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,encoder_hidden_states=SCREAMING_SNAKE_CASE_      ,encoder_attention_mask=SCREAMING_SNAKE_CASE_      ,past_key_values=SCREAMING_SNAKE_CASE_      ,output_hidden_states=SCREAMING_SNAKE_CASE_      ,)['''hidden_states'''][0]

										# select random slice
										snake_case__       :					List[str]										=					ids_tensor((1,)      ,output_from_past.shape[-1]      ).item()
										snake_case__       :					Any										=					output_from_no_past[:, -3:, random_slice_idx].detach()
										snake_case__       :					int										=					output_from_past[:, :, random_slice_idx].detach()

										self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]      )

										# test that outputs are equal for slice
										self.parent.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,atol=1E-3      )      )

				def       _lowerCamelCase					(   self   :     List[str]      ,SCREAMING_SNAKE_CASE_   :     str      ,SCREAMING_SNAKE_CASE_   :     str      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]      ,SCREAMING_SNAKE_CASE_   :     Dict      ,SCREAMING_SNAKE_CASE_   :     List[Any]      ,SCREAMING_SNAKE_CASE_   :     Any      ,SCREAMING_SNAKE_CASE_   :     Tuple      ):
										snake_case__       :					Any										=					RemBertForQuestionAnswering(config=SCREAMING_SNAKE_CASE_      )
										model.to(SCREAMING_SNAKE_CASE_      )
										model.eval()
										snake_case__       :					Dict										=					model(
										    SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      ,start_positions=SCREAMING_SNAKE_CASE_      ,end_positions=SCREAMING_SNAKE_CASE_      ,)
										self.parent.assertEqual(result.start_logits.shape      ,(self.batch_size, self.seq_length)      )
										self.parent.assertEqual(result.end_logits.shape      ,(self.batch_size, self.seq_length)      )

				def       _lowerCamelCase					(   self   :     Any      ,SCREAMING_SNAKE_CASE_   :     Tuple      ,SCREAMING_SNAKE_CASE_   :     Any      ,SCREAMING_SNAKE_CASE_   :     List[str]      ,SCREAMING_SNAKE_CASE_   :     str      ,SCREAMING_SNAKE_CASE_   :     Dict      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]      ):
										snake_case__       :					List[Any]										=					self.num_labels
										snake_case__       :					str										=					RemBertForSequenceClassification(SCREAMING_SNAKE_CASE_      )
										model.to(SCREAMING_SNAKE_CASE_      )
										model.eval()
										snake_case__       :					Optional[int]										=					model(SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      ,labels=SCREAMING_SNAKE_CASE_      )
										self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.num_labels)      )

				def       _lowerCamelCase					(   self   :     List[Any]      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     int      ,SCREAMING_SNAKE_CASE_   :     Optional[int]      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]      ,SCREAMING_SNAKE_CASE_   :     List[str]      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]      ,SCREAMING_SNAKE_CASE_   :     int      ):
										snake_case__       :					List[str]										=					self.num_labels
										snake_case__       :					List[Any]										=					RemBertForTokenClassification(config=SCREAMING_SNAKE_CASE_      )
										model.to(SCREAMING_SNAKE_CASE_      )
										model.eval()
										snake_case__       :					Optional[int]										=					model(SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      ,labels=SCREAMING_SNAKE_CASE_      )
										self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.seq_length, self.num_labels)      )

				def       _lowerCamelCase					(   self   :     str      ,SCREAMING_SNAKE_CASE_   :     int      ,SCREAMING_SNAKE_CASE_   :     int      ,SCREAMING_SNAKE_CASE_   :     int      ,SCREAMING_SNAKE_CASE_   :     int      ,SCREAMING_SNAKE_CASE_   :     int      ,SCREAMING_SNAKE_CASE_   :     Dict      ,SCREAMING_SNAKE_CASE_   :     Union[str, Any]      ):
										snake_case__       :					int										=					self.num_choices
										snake_case__       :					List[Any]										=					RemBertForMultipleChoice(config=SCREAMING_SNAKE_CASE_      )
										model.to(SCREAMING_SNAKE_CASE_      )
										model.eval()
										snake_case__       :					Union[str, Any]										=					input_ids.unsqueeze(1      ).expand(-1      ,self.num_choices      ,-1      ).contiguous()
										snake_case__       :					Tuple										=					token_type_ids.unsqueeze(1      ).expand(-1      ,self.num_choices      ,-1      ).contiguous()
										snake_case__       :					str										=					input_mask.unsqueeze(1      ).expand(-1      ,self.num_choices      ,-1      ).contiguous()
										snake_case__       :					Dict										=					model(
										    SCREAMING_SNAKE_CASE_      ,attention_mask=SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      ,labels=SCREAMING_SNAKE_CASE_      ,)
										self.parent.assertEqual(result.logits.shape      ,(self.batch_size, self.num_choices)      )





				def       _lowerCamelCase					(   self   :     List[str]      ):
										snake_case__       :					int										=					self.prepare_config_and_inputs()
										(
										    (
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	
										)       :					int										=					config_and_inputs
										snake_case__       :					Dict										=					{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
										return config, inputs_dict







@require_torch
class       __lowercase       (			snake_case_						,   snake_case_						,   unittest.TestCase	):


				'''simple docstring'''


				_lowerCAmelCase				=(
				    (
				        RemBertModel,
				        RemBertForMaskedLM,
				        RemBertForCausalLM,
				        RemBertForMultipleChoice,
				        RemBertForQuestionAnswering,
				        RemBertForSequenceClassification,
				        RemBertForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				_lowerCAmelCase				=(RemBertForCausalLM,) if is_torch_available() else ()
				_lowerCAmelCase				=(
				    {
				        "feature-extraction": RemBertModel,
				        "fill-mask": RemBertForMaskedLM,
				        "question-answering": RemBertForQuestionAnswering,
				        "text-classification": RemBertForSequenceClassification,
				        "text-generation": RemBertForCausalLM,
				        "token-classification": RemBertForTokenClassification,
				        "zero-shot": RemBertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)

				def       _lowerCamelCase					(   self   :     List[str]      ):
										snake_case__       :					str										=					RemBertModelTester(self      )
										snake_case__       :					Tuple										=					ConfigTester(self      ,config_class=SCREAMING_SNAKE_CASE_      ,hidden_size=3_7      )

				def       _lowerCamelCase					(   self   :     Union[str, Any]      ):
										self.config_tester.run_common_tests()

				def       _lowerCamelCase					(   self   :     Any      ):
										snake_case__       :					List[str]										=					self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     Union[str, Any]      ):
										snake_case__       :					str										=					self.model_tester.prepare_config_and_inputs()
										for type in ["absolute", "relative_key", "relative_key_query"]:
																snake_case__       :					int										=					type
																self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     Optional[int]      ):
										snake_case__       :					Optional[int]										=					self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     Optional[Any]      ):
										snake_case__       :					Union[str, Any]										=					self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     Any      ):
										snake_case__       :					Tuple										=					self.model_tester.prepare_config_and_inputs_for_decoder()
										self.model_tester.create_and_check_decoder_model_past_large_inputs(*SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     Union[str, Any]      ):
										snake_case__       :					Optional[int]										=					self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     List[str]      ):
										snake_case__       :					Optional[Any]										=					self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     Tuple      ):
										snake_case__       :					List[str]										=					self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     Any      ):
										snake_case__       :					List[Any]										=					self.model_tester.prepare_config_and_inputs_for_decoder()
										self.model_tester.create_and_check_model_as_decoder(*SCREAMING_SNAKE_CASE_      )

				def       _lowerCamelCase					(   self   :     int      ):
										# This regression test was failing with PyTorch < 1.3
										(
										    (
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	(
										    snake_case__
										)			,	
										)       :					int										=					self.model_tester.prepare_config_and_inputs_for_decoder()

										snake_case__       :					List[str]										=					None

										self.model_tester.create_and_check_model_as_decoder(
										    SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,SCREAMING_SNAKE_CASE_      ,)





				@slow
				def       _lowerCamelCase					(   self   :     Tuple      ):
										for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																snake_case__       :					List[Any]										=					RemBertModel.from_pretrained(SCREAMING_SNAKE_CASE_      )
																self.assertIsNotNone(SCREAMING_SNAKE_CASE_      )







@require_torch
class       __lowercase       (			unittest.TestCase	):


				'''simple docstring'''


				@slow
				def       _lowerCamelCase					(   self   :     Dict      ):
										# Test exact values at the last hidden layer
										snake_case__       :					List[Any]										=					RemBertModel.from_pretrained('''google/rembert'''      )
										snake_case__       :					Dict										=					torch.tensor([[3_1_2, 5_6_4_9_8, 3_1_3, 2_1_2_5, 3_1_3]]      )
										snake_case__       :					List[Any]										=					torch.tensor([[0, 0, 0, 1, 1]]      )
										with torch.no_grad():
																snake_case__       :					Any										=					model(SCREAMING_SNAKE_CASE_      ,token_type_ids=SCREAMING_SNAKE_CASE_      ,output_hidden_states=SCREAMING_SNAKE_CASE_      )

										snake_case__       :					List[Any]										=					1_1_5_2

										snake_case__       :					Dict										=					torch.Size((1, 5, hidden_size)      )
										self.assertEqual(output['''last_hidden_state'''].shape      ,SCREAMING_SNAKE_CASE_      )

										snake_case__       :					int										=					torch.tensor(
										    [
										        [
										            [0.0754, -0.2022, 0.1904],
										            [-0.3354, -0.3692, -0.4791],
										            [-0.2314, -0.6729, -0.0749],
										            [-0.0396, -0.3105, -0.4234],
										            [-0.1571, -0.0525, 0.5353],
										        ]
										    ]      )

										# Running on the original tf implementation gives slightly different results here.
										# Not clear why this variations is present
										# TODO: Find reason for discrepancy
										# expected_original_implementation = [[
										#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
										#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
										#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
										#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
										#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
										# ]]

										self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3]      ,SCREAMING_SNAKE_CASE_      ,atol=1E-4      )      )








"""simple docstring"""





import argparse
import os
import sys
import urllib.request
import zipfile


_lowercase	      =    ["""CoLA""", """SST""", """MRPC""", """QQP""", """STS""", """MNLI""", """SNLI""", """QNLI""", """RTE""", """WNLI""", """diagnostic"""]
_lowercase	      =    {
    """CoLA""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FCoLA.zip?alt=media&token=46d5e637-3411-4188-bc44-5809b5bfb5f4""",
    """SST""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8""",
    """MRPC""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc""",
    """QQP""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQQP.zip?alt=media&token=700c6acf-160d-4d89-81d1-de4191d02cb5""",
    """STS""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSTS-B.zip?alt=media&token=bddb94a7-8706-4e0d-a694-1109e12273b5""",
    """MNLI""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FMNLI.zip?alt=media&token=50329ea1-e339-40e2-809c-10c40afff3ce""",
    """SNLI""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSNLI.zip?alt=media&token=4afcfbb2-ff0c-4b2d-a09a-dbf07926f4df""",
    """QNLI""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQNLIv2.zip?alt=media&token=6fdcf570-0fc5-4631-8456-9505272d1601""",
    """RTE""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FRTE.zip?alt=media&token=5efa7e85-a0bb-4f19-8ea2-9e1840f077fb""",
    """WNLI""": """https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FWNLI.zip?alt=media&token=068ad0a0-ded7-4bd7-99a5-5e00222e0faf""",
    """diagnostic""": """https://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D""",
}

_lowercase	      =    """https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt"""
_lowercase	      =    """https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt"""

def 		lowerCAmelCase							( lowerCamelCase      ,			lowerCamelCase		):
						'''simple docstring'''


						print(f'''Downloading and extracting {task}...'''		)
						snake_case__       :					Optional[Any]										=					f'''{task}.zip'''
						urllib.request.urlretrieve(TASKaPATH[task]      ,			lowerCamelCase		)
						with zipfile.ZipFile(lowerCamelCase		) as zip_ref:
												zip_ref.extractall(lowerCamelCase		)
						os.remove(lowerCamelCase		)
						print('''\tCompleted!'''		)

def 		lowerCAmelCase							( lowerCamelCase      ,			lowerCamelCase		):
						'''simple docstring'''


						print('''Processing MRPC...'''		)
						snake_case__       :					Union[str, Any]										=					os.path.join(lowerCamelCase      ,			'''MRPC'''		)
						if not os.path.isdir(lowerCamelCase		):
												os.mkdir(lowerCamelCase		)
						if path_to_data:
												snake_case__       :					List[Any]										=					os.path.join(lowerCamelCase      ,			'''msr_paraphrase_train.txt'''		)
												snake_case__       :					Any										=					os.path.join(lowerCamelCase      ,			'''msr_paraphrase_test.txt'''		)
						else:
												print('''Local MRPC data not specified, downloading data from %s''' % MRPC_TRAIN		)
												snake_case__       :					Tuple										=					os.path.join(lowerCamelCase      ,			'''msr_paraphrase_train.txt'''		)
												snake_case__       :					Optional[int]										=					os.path.join(lowerCamelCase      ,			'''msr_paraphrase_test.txt'''		)
												urllib.request.urlretrieve(lowerCamelCase      ,			lowerCamelCase		)
												urllib.request.urlretrieve(lowerCamelCase      ,			lowerCamelCase		)
						if not os.path.isfile(lowerCamelCase		):
												raise ValueError(f'''Train data not found at {mrpc_train_file}'''		)
						if not os.path.isfile(lowerCamelCase		):
												raise ValueError(f'''Test data not found at {mrpc_test_file}'''		)
						urllib.request.urlretrieve(TASKaPATH['''MRPC''']      ,			os.path.join(lowerCamelCase      ,			'''dev_ids.tsv'''		)		)

						snake_case__       :					Optional[Any]										=					[]
						with open(os.path.join(lowerCamelCase      ,			'''dev_ids.tsv'''		)      ,			encoding='''utf8'''		) as ids_fh:
												for row in ids_fh:
																		dev_ids.append(row.strip().split('''\t'''		)		)

						with open(lowerCamelCase      ,			encoding='''utf8'''		) as data_fh, open(
						    os.path.join(lowerCamelCase      ,			'''train.tsv'''		)      ,			'''w'''      ,			encoding='''utf8'''		) as train_fh, open(os.path.join(lowerCamelCase      ,			'''dev.tsv'''		)      ,			'''w'''      ,			encoding='''utf8'''		) as dev_fh:
												snake_case__       :					Any										=					data_fh.readline()
												train_fh.write(lowerCamelCase		)
												dev_fh.write(lowerCamelCase		)
												for row in data_fh:
																		snake_case__			,	snake_case__			,	snake_case__			,	snake_case__			,	snake_case__       :					Dict										=					row.strip().split('''\t'''		)
																		if [ida, ida] in dev_ids:
																								dev_fh.write('''%s\t%s\t%s\t%s\t%s\n''' % (label, ida, ida, sa, sa)		)
																		else:
																								train_fh.write('''%s\t%s\t%s\t%s\t%s\n''' % (label, ida, ida, sa, sa)		)

						with open(lowerCamelCase      ,			encoding='''utf8'''		) as data_fh, open(
						    os.path.join(lowerCamelCase      ,			'''test.tsv'''		)      ,			'''w'''      ,			encoding='''utf8'''		) as test_fh:
												snake_case__       :					Tuple										=					data_fh.readline()
												test_fh.write('''index\t#1 ID\t#2 ID\t#1 String\t#2 String\n'''		)
												for idx, row in enumerate(lowerCamelCase		):
																		snake_case__			,	snake_case__			,	snake_case__			,	snake_case__			,	snake_case__       :					Union[str, Any]										=					row.strip().split('''\t'''		)
																		test_fh.write('''%d\t%s\t%s\t%s\t%s\n''' % (idx, ida, ida, sa, sa)		)
						print('''\tCompleted!'''		)

def 		lowerCAmelCase							( lowerCamelCase		):
						'''simple docstring'''


						print('''Downloading and extracting diagnostic...'''		)
						if not os.path.isdir(os.path.join(lowerCamelCase      ,			'''diagnostic'''		)		):
												os.mkdir(os.path.join(lowerCamelCase      ,			'''diagnostic'''		)		)
						snake_case__       :					Any										=					os.path.join(lowerCamelCase      ,			'''diagnostic'''      ,			'''diagnostic.tsv'''		)
						urllib.request.urlretrieve(TASKaPATH['''diagnostic''']      ,			lowerCamelCase		)
						print('''\tCompleted!'''		)
						return

def 		lowerCAmelCase							( lowerCamelCase		):
						'''simple docstring'''


						snake_case__       :					Any										=					task_names.split(''','''		)
						if "all" in task_names:
												snake_case__       :					str										=					TASKS
						else:
												snake_case__       :					Union[str, Any]										=					[]
												for task_name in task_names:
																		if task_name not in TASKS:
																								raise ValueError(f'''Task {task_name} not found!'''		)
																		tasks.append(lowerCamelCase		)
						return tasks

def 		lowerCAmelCase							( lowerCamelCase		):
						'''simple docstring'''


						snake_case__       :					int										=					argparse.ArgumentParser()
						parser.add_argument('''--data_dir'''      ,			help='''directory to save data to'''      ,			type=lowerCamelCase      ,			default='''glue_data'''		)
						parser.add_argument(
						    '''--tasks'''      ,			help='''tasks to download data for as a comma separated string'''      ,			type=lowerCamelCase      ,			default='''all'''		)
						parser.add_argument(
						    '''--path_to_mrpc'''      ,			help='''path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt'''      ,			type=lowerCamelCase      ,			default=''''''      ,			)
						snake_case__       :					Dict										=					parser.parse_args(lowerCamelCase		)

						if not os.path.isdir(args.data_dir		):
												os.mkdir(args.data_dir		)
						snake_case__       :					Union[str, Any]										=					get_tasks(args.tasks		)

						for task in tasks:
												if task == "MRPC":
																		format_mrpc(args.data_dir      ,			args.path_to_mrpc		)
												elif task == "diagnostic":
																		download_diagnostic(args.data_dir		)
												else:
																		download_and_extract(lowerCamelCase      ,			args.data_dir		)


if __name__ == "__main__":
							sys.exit(main(sys.argv[1:]))


