

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      RemBertConfig,
      RemBertForCausalLM,
      RemBertForMaskedLM,
      RemBertForMultipleChoice,
      RemBertForQuestionAnswering,
      RemBertForSequenceClassification,
      RemBertForTokenClassification,
      RemBertModel,
  )
  from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class 							a__							:







  """simple docstring"""




  def __init__(      self     :			List[Any]					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			List[Any]=1_3					,					UpperCAmelCase__     :			Union[str, Any]=7					,					UpperCAmelCase__     :			Any=True					,					UpperCAmelCase__     :			List[str]=True					,					UpperCAmelCase__     :			int=True					,					UpperCAmelCase__     :			str=True					,					UpperCAmelCase__     :			str=9_9					,					UpperCAmelCase__     :			Dict=3_2					,					UpperCAmelCase__     :			List[Any]=1_8					,					UpperCAmelCase__     :			Optional[Any]=4_3					,					UpperCAmelCase__     :			List[str]=5					,					UpperCAmelCase__     :			str=4					,					UpperCAmelCase__     :			Union[str, Any]=3_7					,					UpperCAmelCase__     :			str="gelu"					,					UpperCAmelCase__     :			Dict=0.1					,					UpperCAmelCase__     :			int=0.1					,					UpperCAmelCase__     :			Dict=5_1_2					,					UpperCAmelCase__     :			Optional[int]=1_6					,					UpperCAmelCase__     :			Dict=2					,					UpperCAmelCase__     :			str=0.02					,					UpperCAmelCase__     :			Optional[int]=3					,					UpperCAmelCase__     :			Optional[Any]=4					,					UpperCAmelCase__     :			Dict=None					,					)       ->Union[str, Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Tuple	  = parent
        SCREAMING_SNAKE_CASE			:   Any	  = batch_size
        SCREAMING_SNAKE_CASE			:   List[str]	  = seq_length
        SCREAMING_SNAKE_CASE			:   Dict	  = is_training
        SCREAMING_SNAKE_CASE			:   List[str]	  = use_input_mask
        SCREAMING_SNAKE_CASE			:   Tuple	  = use_token_type_ids
        SCREAMING_SNAKE_CASE			:   List[str]	  = use_labels
        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = vocab_size
        SCREAMING_SNAKE_CASE			:   Any	  = hidden_size
        SCREAMING_SNAKE_CASE			:   int	  = input_embedding_size
        SCREAMING_SNAKE_CASE			:   List[str]	  = output_embedding_size
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = num_hidden_layers
        SCREAMING_SNAKE_CASE			:   str	  = num_attention_heads
        SCREAMING_SNAKE_CASE			:   Dict	  = intermediate_size
        SCREAMING_SNAKE_CASE			:   Dict	  = hidden_act
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = hidden_dropout_prob
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = attention_probs_dropout_prob
        SCREAMING_SNAKE_CASE			:   Tuple	  = max_position_embeddings
        SCREAMING_SNAKE_CASE			:   List[str]	  = type_vocab_size
        SCREAMING_SNAKE_CASE			:   List[str]	  = type_sequence_label_size
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = initializer_range
        SCREAMING_SNAKE_CASE			:   Dict	  = num_labels
        SCREAMING_SNAKE_CASE			:   Optional[int]	  = num_choices
        SCREAMING_SNAKE_CASE			:   List[str]	  = scope


  def 			_lowercase   (      self     :			Any		)       ->Any:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Optional[int]	  = ids_tensor([self.batch_size, self.seq_length]					,					self.vocab_size		)

        SCREAMING_SNAKE_CASE			:   Any	  = None
        if self.use_input_mask:
              SCREAMING_SNAKE_CASE			:   str	  = random_attention_mask([self.batch_size, self.seq_length]		)

        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = None
        if self.use_token_type_ids:
              SCREAMING_SNAKE_CASE			:   Optional[int]	  = ids_tensor([self.batch_size, self.seq_length]					,					self.type_vocab_size		)

        SCREAMING_SNAKE_CASE			:   int	  = None
        SCREAMING_SNAKE_CASE			:   Any	  = None
        SCREAMING_SNAKE_CASE			:   Tuple	  = None
        if self.use_labels:
              SCREAMING_SNAKE_CASE			:   List[str]	  = ids_tensor([self.batch_size]					,					self.type_sequence_label_size		)
              SCREAMING_SNAKE_CASE			:   int	  = ids_tensor([self.batch_size, self.seq_length]					,					self.num_labels		)
              SCREAMING_SNAKE_CASE			:   str	  = ids_tensor([self.batch_size]					,					self.num_choices		)

        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = RemBertConfig(
            vocab_size=self.vocab_size					,					hidden_size=self.hidden_size					,					input_embedding_size=self.input_embedding_size					,					output_embedding_size=self.output_embedding_size					,					num_hidden_layers=self.num_hidden_layers					,					num_attention_heads=self.num_attention_heads					,					intermediate_size=self.intermediate_size					,					hidden_act=self.hidden_act					,					hidden_dropout_prob=self.hidden_dropout_prob					,					attention_probs_dropout_prob=self.attention_probs_dropout_prob					,					max_position_embeddings=self.max_position_embeddings					,					type_vocab_size=self.type_vocab_size					,					is_decoder=UpperCAmelCase__					,					initializer_range=self.initializer_range					,					)

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


  def 			_lowercase   (      self     :			int		)       ->Dict:
        """simple docstring"""







        (
            (
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						
        )			:   List[Any]	  = self.prepare_config_and_inputs()

        SCREAMING_SNAKE_CASE			:   Dict	  = True
        SCREAMING_SNAKE_CASE			:   Tuple	  = floats_tensor([self.batch_size, self.seq_length, self.hidden_size]		)
        SCREAMING_SNAKE_CASE			:   Tuple	  = ids_tensor([self.batch_size, self.seq_length]					,					vocab_size=2		)

        return (
            config,
            input_ids,
            token_type_ids,
            input_mask,
            sequence_labels,
            token_labels,
            choice_labels,
            encoder_hidden_states,
            encoder_attention_mask,
        )


  def 			_lowercase   (      self     :			Dict					,					UpperCAmelCase__     :			List[str]					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			Optional[int]					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			Optional[int]		)       ->Union[str, Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = RemBertModel(config=UpperCAmelCase__		)
        model.to(UpperCAmelCase__		)
        model.eval()
        SCREAMING_SNAKE_CASE			:   Any	  = model(UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE			:   List[str]	  = model(UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__		)
        SCREAMING_SNAKE_CASE			:   Tuple	  = model(UpperCAmelCase__		)
        self.parent.assertEqual(result.last_hidden_state.shape					,					(self.batch_size, self.seq_length, self.hidden_size)		)


  def 			_lowercase   (      self     :			List[str]					,					UpperCAmelCase__     :			List[Any]					,					UpperCAmelCase__     :			List[Any]					,					UpperCAmelCase__     :			Any					,					UpperCAmelCase__     :			List[str]					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			Dict					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			List[Any]					,					)       ->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   str	  = True
        SCREAMING_SNAKE_CASE			:   str	  = RemBertModel(UpperCAmelCase__		)
        model.to(UpperCAmelCase__		)
        model.eval()
        SCREAMING_SNAKE_CASE			:   Tuple	  = model(
            UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__					,					encoder_hidden_states=UpperCAmelCase__					,					encoder_attention_mask=UpperCAmelCase__					,					)
        SCREAMING_SNAKE_CASE			:   Optional[int]	  = model(
            UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__					,					encoder_hidden_states=UpperCAmelCase__					,					)
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = model(UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__		)
        self.parent.assertEqual(result.last_hidden_state.shape					,					(self.batch_size, self.seq_length, self.hidden_size)		)


  def 			_lowercase   (      self     :			Any					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			List[str]					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			Any					,					UpperCAmelCase__     :			Optional[int]					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			int					,					)       ->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   str	  = RemBertForCausalLM(config=UpperCAmelCase__		)
        model.to(UpperCAmelCase__		)
        model.eval()
        SCREAMING_SNAKE_CASE			:   Tuple	  = model(UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__					,					labels=UpperCAmelCase__		)
        self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.seq_length, self.vocab_size)		)


  def 			_lowercase   (      self     :			Tuple					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			Optional[Any]					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			List[str]					,					UpperCAmelCase__     :			Dict					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			Union[str, Any]		)       ->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   List[Any]	  = RemBertForMaskedLM(config=UpperCAmelCase__		)
        model.to(UpperCAmelCase__		)
        model.eval()
        SCREAMING_SNAKE_CASE			:   Tuple	  = model(UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__					,					labels=UpperCAmelCase__		)
        self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.seq_length, self.vocab_size)		)


  def 			_lowercase   (      self     :			Dict					,					UpperCAmelCase__     :			Optional[Any]					,					UpperCAmelCase__     :			List[Any]					,					UpperCAmelCase__     :			Optional[int]					,					UpperCAmelCase__     :			Dict					,					UpperCAmelCase__     :			Any					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			Dict					,					UpperCAmelCase__     :			Tuple					,					)       ->List[str]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   List[Any]	  = True
        SCREAMING_SNAKE_CASE			:   int	  = True
        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = RemBertForCausalLM(config=UpperCAmelCase__		)
        model.to(UpperCAmelCase__		)
        model.eval()

        # first forward pass
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = model(
            UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					encoder_hidden_states=UpperCAmelCase__					,					encoder_attention_mask=UpperCAmelCase__					,					use_cache=UpperCAmelCase__					,					)
        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = outputs.past_key_values

        # create hypothetical multiple next token and extent to next_input_ids
        SCREAMING_SNAKE_CASE			:   List[str]	  = ids_tensor((self.batch_size, 3)					,					config.vocab_size		)
        SCREAMING_SNAKE_CASE			:   List[str]	  = ids_tensor((self.batch_size, 3)					,					vocab_size=2		)

        # append to next input_ids and
        SCREAMING_SNAKE_CASE			:   int	  = torch.cat([input_ids, next_tokens]					,					dim=-1		)
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = torch.cat([input_mask, next_mask]					,					dim=-1		)

        SCREAMING_SNAKE_CASE			:   List[str]	  = model(
            UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					encoder_hidden_states=UpperCAmelCase__					,					encoder_attention_mask=UpperCAmelCase__					,					output_hidden_states=UpperCAmelCase__					,					)["""hidden_states"""][0]
        SCREAMING_SNAKE_CASE			:   Dict	  = model(
            UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					encoder_hidden_states=UpperCAmelCase__					,					encoder_attention_mask=UpperCAmelCase__					,					past_key_values=UpperCAmelCase__					,					output_hidden_states=UpperCAmelCase__					,					)["""hidden_states"""][0]

        # select random slice
        SCREAMING_SNAKE_CASE			:   Any	  = ids_tensor((1,)					,					output_from_past.shape[-1]		).item()
        SCREAMING_SNAKE_CASE			:   List[str]	  = output_from_no_past[:, -3:, random_slice_idx].detach()
        SCREAMING_SNAKE_CASE			:   Optional[int]	  = output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]		)

        # test that outputs are equal for slice
        self.parent.assertTrue(torch.allclose(UpperCAmelCase__					,					UpperCAmelCase__					,					atol=1e-3		)		)


  def 			_lowercase   (      self     :			Optional[int]					,					UpperCAmelCase__     :			Tuple					,					UpperCAmelCase__     :			Optional[int]					,					UpperCAmelCase__     :			Optional[int]					,					UpperCAmelCase__     :			Tuple					,					UpperCAmelCase__     :			Optional[Any]					,					UpperCAmelCase__     :			List[str]					,					UpperCAmelCase__     :			List[Any]		)       ->Any:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   str	  = RemBertForQuestionAnswering(config=UpperCAmelCase__		)
        model.to(UpperCAmelCase__		)
        model.eval()
        SCREAMING_SNAKE_CASE			:   int	  = model(
            UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__					,					start_positions=UpperCAmelCase__					,					end_positions=UpperCAmelCase__					,					)
        self.parent.assertEqual(result.start_logits.shape					,					(self.batch_size, self.seq_length)		)
        self.parent.assertEqual(result.end_logits.shape					,					(self.batch_size, self.seq_length)		)


  def 			_lowercase   (      self     :			List[str]					,					UpperCAmelCase__     :			Any					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			Optional[Any]					,					UpperCAmelCase__     :			Optional[Any]					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			str					,					UpperCAmelCase__     :			Optional[Any]		)       ->Optional[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Optional[int]	  = self.num_labels
        SCREAMING_SNAKE_CASE			:   int	  = RemBertForSequenceClassification(UpperCAmelCase__		)
        model.to(UpperCAmelCase__		)
        model.eval()
        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = model(UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__					,					labels=UpperCAmelCase__		)
        self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.num_labels)		)


  def 			_lowercase   (      self     :			Tuple					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			List[str]					,					UpperCAmelCase__     :			Union[str, Any]					,					UpperCAmelCase__     :			Tuple					,					UpperCAmelCase__     :			Tuple					,					UpperCAmelCase__     :			Optional[Any]					,					UpperCAmelCase__     :			Union[str, Any]		)       ->Tuple:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   int	  = self.num_labels
        SCREAMING_SNAKE_CASE			:   List[str]	  = RemBertForTokenClassification(config=UpperCAmelCase__		)
        model.to(UpperCAmelCase__		)
        model.eval()
        SCREAMING_SNAKE_CASE			:   str	  = model(UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__					,					labels=UpperCAmelCase__		)
        self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.seq_length, self.num_labels)		)


  def 			_lowercase   (      self     :			List[Any]					,					UpperCAmelCase__     :			List[str]					,					UpperCAmelCase__     :			int					,					UpperCAmelCase__     :			Dict					,					UpperCAmelCase__     :			List[Any]					,					UpperCAmelCase__     :			Any					,					UpperCAmelCase__     :			Any					,					UpperCAmelCase__     :			Tuple		)       ->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Dict	  = self.num_choices
        SCREAMING_SNAKE_CASE			:   Any	  = RemBertForMultipleChoice(config=UpperCAmelCase__		)
        model.to(UpperCAmelCase__		)
        model.eval()
        SCREAMING_SNAKE_CASE			:   Any	  = input_ids.unsqueeze(1		).expand(-1					,					self.num_choices					,					-1		).contiguous()
        SCREAMING_SNAKE_CASE			:   List[Any]	  = token_type_ids.unsqueeze(1		).expand(-1					,					self.num_choices					,					-1		).contiguous()
        SCREAMING_SNAKE_CASE			:   List[str]	  = input_mask.unsqueeze(1		).expand(-1					,					self.num_choices					,					-1		).contiguous()
        SCREAMING_SNAKE_CASE			:   List[str]	  = model(
            UpperCAmelCase__					,					attention_mask=UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__					,					labels=UpperCAmelCase__					,					)
        self.parent.assertEqual(result.logits.shape					,					(self.batch_size, self.num_choices)		)




  def 			_lowercase   (      self     :			List[str]		)       ->Dict:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Tuple	  = self.prepare_config_and_inputs()
        (
            (
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						
        )			:   List[Any]	  = config_and_inputs
        SCREAMING_SNAKE_CASE			:   List[Any]	  = {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
        return config, inputs_dict







@require_torch
class 							a__							( UpperCAmelCase						,      UpperCAmelCase						,      unittest.TestCase							):







  """simple docstring"""




  UpperCAmelCase__   :			int      							=(
      (
          RemBertModel,
          RemBertForMaskedLM,
          RemBertForCausalLM,
          RemBertForMultipleChoice,
          RemBertForQuestionAnswering,
          RemBertForSequenceClassification,
          RemBertForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  UpperCAmelCase__   :			Optional[Any]      							=(RemBertForCausalLM,) if is_torch_available() else ()
  UpperCAmelCase__   :			Dict      							=(
      {
          """feature-extraction""": RemBertModel,
          """fill-mask""": RemBertForMaskedLM,
          """question-answering""": RemBertForQuestionAnswering,
          """text-classification""": RemBertForSequenceClassification,
          """text-generation""": RemBertForCausalLM,
          """token-classification""": RemBertForTokenClassification,
          """zero-shot""": RemBertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )


  def 			_lowercase   (      self     :			Any		)       ->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Any	  = RemBertModelTester(self		)
        SCREAMING_SNAKE_CASE			:   int	  = ConfigTester(self					,					config_class=UpperCAmelCase__					,					hidden_size=3_7		)


  def 			_lowercase   (      self     :			Dict		)       ->Tuple:
        """simple docstring"""







        self.config_tester.run_common_tests()


  def 			_lowercase   (      self     :			Union[str, Any]		)       ->Union[str, Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   List[str]	  = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*UpperCAmelCase__		)


  def 			_lowercase   (      self     :			int		)       ->Dict:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   str	  = self.model_tester.prepare_config_and_inputs()
        for type in ["absolute", "relative_key", "relative_key_query"]:
              SCREAMING_SNAKE_CASE			:   str	  = type
              self.model_tester.create_and_check_model(*UpperCAmelCase__		)


  def 			_lowercase   (      self     :			Optional[int]		)       ->Optional[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Tuple	  = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase__		)


  def 			_lowercase   (      self     :			List[Any]		)       ->Dict:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Tuple	  = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase__		)


  def 			_lowercase   (      self     :			str		)       ->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   List[str]	  = self.model_tester.prepare_config_and_inputs_for_decoder()
        self.model_tester.create_and_check_decoder_model_past_large_inputs(*UpperCAmelCase__		)


  def 			_lowercase   (      self     :			Dict		)       ->List[str]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Any	  = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase__		)


  def 			_lowercase   (      self     :			Union[str, Any]		)       ->List[Any]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Any	  = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase__		)


  def 			_lowercase   (      self     :			Optional[int]		)       ->Tuple:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Optional[Any]	  = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase__		)


  def 			_lowercase   (      self     :			int		)       ->Optional[int]:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Dict	  = self.model_tester.prepare_config_and_inputs_for_decoder()
        self.model_tester.create_and_check_model_as_decoder(*UpperCAmelCase__		)


  def 			_lowercase   (      self     :			Dict		)       ->Any:
        """simple docstring"""







        (
            (
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						(
            SCREAMING_SNAKE_CASE
        )  ,						
        )			:   List[Any]	  = self.model_tester.prepare_config_and_inputs_for_decoder()

        SCREAMING_SNAKE_CASE			:   Tuple	  = None

        self.model_tester.create_and_check_model_as_decoder(
            UpperCAmelCase__					,					UpperCAmelCase__					,					UpperCAmelCase__					,					UpperCAmelCase__					,					UpperCAmelCase__					,					UpperCAmelCase__					,					UpperCAmelCase__					,					UpperCAmelCase__					,					UpperCAmelCase__					,					)




  @slow
  def 			_lowercase   (      self     :			Optional[Any]		)       ->int:
        """simple docstring"""







        for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              SCREAMING_SNAKE_CASE			:   Optional[Any]	  = RemBertModel.from_pretrained(UpperCAmelCase__		)
              self.assertIsNotNone(UpperCAmelCase__		)







@require_torch
class 							a__							( unittest.TestCase							):







  """simple docstring"""




  @slow
  def 			_lowercase   (      self     :			str		)       ->Any:
        """simple docstring"""







        SCREAMING_SNAKE_CASE			:   Dict	  = RemBertModel.from_pretrained("""google/rembert"""		)
        SCREAMING_SNAKE_CASE			:   List[str]	  = torch.tensor([[3_1_2, 5_6_4_9_8, 3_1_3, 2_1_2_5, 3_1_3]]		)
        SCREAMING_SNAKE_CASE			:   Dict	  = torch.tensor([[0, 0, 0, 1, 1]]		)
        with torch.no_grad():
              SCREAMING_SNAKE_CASE			:   List[str]	  = model(UpperCAmelCase__					,					token_type_ids=UpperCAmelCase__					,					output_hidden_states=UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE			:   Optional[int]	  = 1_1_5_2

        SCREAMING_SNAKE_CASE			:   Union[str, Any]	  = torch.Size((1, 5, hidden_size)		)
        self.assertEqual(output["""last_hidden_state"""].shape					,					UpperCAmelCase__		)

        SCREAMING_SNAKE_CASE			:   List[Any]	  = torch.tensor(
            [
                [
                    [0.07_54, -0.20_22, 0.19_04],
                    [-0.33_54, -0.36_92, -0.47_91],
                    [-0.23_14, -0.67_29, -0.07_49],
                    [-0.03_96, -0.31_05, -0.42_34],
                    [-0.15_71, -0.05_25, 0.53_53],
                ]
            ]		)

        # Running on the original tf implementation gives slightly different results here.
        # Not clear why this variations is present
        # TODO: Find reason for discrepancy
        # expected_original_implementation = [[
        #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
        #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
        #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
        #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
        #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
        # ]]

        self.assertTrue(torch.allclose(output["""last_hidden_state"""][:, :, :3]					,					UpperCAmelCase__					,					atol=1e-4		)		)







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCAmelCase__			:  Optional[Any]					     =     {
    """configuration_musicgen""": [
        """MUSICGEN_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """MusicgenConfig""",
        """MusicgenDecoderConfig""",
    ],
    """processing_musicgen""": ["""MusicgenProcessor"""],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase__			:  Any					     =     [
      """MUSICGEN_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """MusicgenForConditionalGeneration""",
      """MusicgenForCausalLM""",
      """MusicgenModel""",
      """MusicgenPreTrainedModel""",
  ]

if TYPE_CHECKING:
  from .configuration_musicgen import (
      MUSICGEN_PRETRAINED_CONFIG_ARCHIVE_MAP,
      MusicgenConfig,
      MusicgenDecoderConfig,
  )
  from .processing_musicgen import MusicgenProcessor

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_musicgen import (
        MUSICGEN_PRETRAINED_MODEL_ARCHIVE_LIST,
        MusicgenForCausalLM,
        MusicgenForConditionalGeneration,
        MusicgenModel,
        MusicgenPreTrainedModel,
    )

else:
  import sys

  UpperCAmelCase__			:  Dict					     =     _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




