






import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import (
     RemBertConfig,
     RemBertForCausalLM,
     RemBertForMaskedLM,
     RemBertForMultipleChoice,
     RemBertForQuestionAnswering,
     RemBertForSequenceClassification,
     RemBertForTokenClassification,
     RemBertModel,
 )
 from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST
class        __snake_case:



 def __init__(	self   :   int ,		UpperCAmelCase__   :   Union[str, Any] ,		UpperCAmelCase__   :   Dict=13 ,		UpperCAmelCase__   :   Union[str, Any]=7 ,		UpperCAmelCase__   :   Any=True ,		UpperCAmelCase__   :   int=True ,		UpperCAmelCase__   :   int=True ,		UpperCAmelCase__   :   int=True ,		UpperCAmelCase__   :   Union[str, Any]=99 ,		UpperCAmelCase__   :   Any=32 ,		UpperCAmelCase__   :   int=18 ,		UpperCAmelCase__   :   Optional[Any]=43 ,		UpperCAmelCase__   :   int=5 ,		UpperCAmelCase__   :   str=4 ,		UpperCAmelCase__   :   Tuple=37 ,		UpperCAmelCase__   :   Any="gelu" ,		UpperCAmelCase__   :   Optional[int]=0.1 ,		UpperCAmelCase__   :   Optional[Any]=0.1 ,		UpperCAmelCase__   :   Optional[Any]=512 ,		UpperCAmelCase__   :   Optional[int]=16 ,		UpperCAmelCase__   :   Any=2 ,		UpperCAmelCase__   :   Union[str, Any]=0.02 ,		UpperCAmelCase__   :   Union[str, Any]=3 ,		UpperCAmelCase__   :   Tuple=4 ,		UpperCAmelCase__   :   int=None ,		):

    """simple docstring"""






    lowercase__						:	int         = parent
    lowercase__						:	int         = batch_size
    lowercase__						:	List[str]         = seq_length
    lowercase__						:	Optional[int]         = is_training
    lowercase__						:	int         = use_input_mask
    lowercase__						:	List[str]         = use_token_type_ids
    lowercase__						:	Union[str, Any]         = use_labels
    lowercase__						:	Tuple         = vocab_size
    lowercase__						:	int         = hidden_size
    lowercase__						:	Tuple         = input_embedding_size
    lowercase__						:	Any         = output_embedding_size
    lowercase__						:	Tuple         = num_hidden_layers
    lowercase__						:	Dict         = num_attention_heads
    lowercase__						:	Union[str, Any]         = intermediate_size
    lowercase__						:	int         = hidden_act
    lowercase__						:	str         = hidden_dropout_prob
    lowercase__						:	str         = attention_probs_dropout_prob
    lowercase__						:	Tuple         = max_position_embeddings
    lowercase__						:	Optional[Any]         = type_vocab_size
    lowercase__						:	List[str]         = type_sequence_label_size
    lowercase__						:	int         = initializer_range
    lowercase__						:	List[str]         = num_labels
    lowercase__						:	List[Any]         = num_choices
    lowercase__						:	Tuple         = scope



 def 						UpperCAmelCase    (	self   :   str	):

    """simple docstring"""






    lowercase__						:	Union[str, Any]         = ids_tensor([self.batch_size, self.seq_length] ,		self.vocab_size	)

    lowercase__						:	int         = None
    if self.use_input_mask:
       lowercase__						:	str         = random_attention_mask([self.batch_size, self.seq_length]	)

    lowercase__						:	str         = None
    if self.use_token_type_ids:
       lowercase__						:	str         = ids_tensor([self.batch_size, self.seq_length] ,		self.type_vocab_size	)

    lowercase__						:	Union[str, Any]         = None
    lowercase__						:	Union[str, Any]         = None
    lowercase__						:	List[str]         = None
    if self.use_labels:
       lowercase__						:	str         = ids_tensor([self.batch_size] ,		self.type_sequence_label_size	)
       lowercase__						:	str         = ids_tensor([self.batch_size, self.seq_length] ,		self.num_labels	)
       lowercase__						:	List[str]         = ids_tensor([self.batch_size] ,		self.num_choices	)

    lowercase__						:	Optional[int]         = RemBertConfig(
        vocab_size=self.vocab_size ,		hidden_size=self.hidden_size ,		input_embedding_size=self.input_embedding_size ,		output_embedding_size=self.output_embedding_size ,		num_hidden_layers=self.num_hidden_layers ,		num_attention_heads=self.num_attention_heads ,		intermediate_size=self.intermediate_size ,		hidden_act=self.hidden_act ,		hidden_dropout_prob=self.hidden_dropout_prob ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob ,		max_position_embeddings=self.max_position_embeddings ,		type_vocab_size=self.type_vocab_size ,		is_decoder=UpperCAmelCase__ ,		initializer_range=self.initializer_range ,		)

    return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



 def 						UpperCAmelCase    (	self   :   Optional[Any]	):

    """simple docstring"""






    (
        lowercase__			
    )						:	int         = self.prepare_config_and_inputs()

    lowercase__						:	List[Any]         = True
    lowercase__						:	Optional[int]         = floats_tensor([self.batch_size, self.seq_length, self.hidden_size]	)
    lowercase__						:	Optional[Any]         = ids_tensor([self.batch_size, self.seq_length] ,		vocab_size=2	)

    return (
        config,
        input_ids,
        token_type_ids,
        input_mask,
        sequence_labels,
        token_labels,
        choice_labels,
        encoder_hidden_states,
        encoder_attention_mask,
    )



 def 						UpperCAmelCase    (	self   :   str ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   Union[str, Any] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Optional[int]	):

    """simple docstring"""






    lowercase__						:	str         = RemBertModel(config=UpperCAmelCase__	)
    model.to(UpperCAmelCase__	)
    model.eval()
    lowercase__						:	Any         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__	)
    lowercase__						:	str         = model(UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__	)
    lowercase__						:	Union[str, Any]         = model(UpperCAmelCase__	)
    self.parent.assertEqual(result.last_hidden_state.shape ,		(self.batch_size, self.seq_length, self.hidden_size)	)



 def 						UpperCAmelCase    (	self   :   List[str] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   List[str] ,		):

    """simple docstring"""






    lowercase__						:	Optional[int]         = True
    lowercase__						:	Union[str, Any]         = RemBertModel(UpperCAmelCase__	)
    model.to(UpperCAmelCase__	)
    model.eval()
    lowercase__						:	Tuple         = model(
        UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		encoder_hidden_states=UpperCAmelCase__ ,		encoder_attention_mask=UpperCAmelCase__ ,		)
    lowercase__						:	int         = model(
        UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		encoder_hidden_states=UpperCAmelCase__ ,		)
    lowercase__						:	int         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__	)
    self.parent.assertEqual(result.last_hidden_state.shape ,		(self.batch_size, self.seq_length, self.hidden_size)	)



 def 						UpperCAmelCase    (	self   :   int ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   Optional[Any] ,		):

    """simple docstring"""






    lowercase__						:	Any         = RemBertForCausalLM(config=UpperCAmelCase__	)
    model.to(UpperCAmelCase__	)
    model.eval()
    lowercase__						:	Union[str, Any]         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		labels=UpperCAmelCase__	)
    self.parent.assertEqual(result.logits.shape ,		(self.batch_size, self.seq_length, self.vocab_size)	)



 def 						UpperCAmelCase    (	self   :   Any ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   List[Any]	):

    """simple docstring"""






    lowercase__						:	str         = RemBertForMaskedLM(config=UpperCAmelCase__	)
    model.to(UpperCAmelCase__	)
    model.eval()
    lowercase__						:	int         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		labels=UpperCAmelCase__	)
    self.parent.assertEqual(result.logits.shape ,		(self.batch_size, self.seq_length, self.vocab_size)	)



 def 						UpperCAmelCase    (	self   :   int ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   List[str] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   int ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Union[str, Any] ,		UpperCAmelCase__   :   Dict ,		):

    """simple docstring"""






    lowercase__						:	List[Any]         = True
    lowercase__						:	Tuple         = True
    lowercase__						:	List[str]         = RemBertForCausalLM(config=UpperCAmelCase__	)
    model.to(UpperCAmelCase__	)
    model.eval()

    # first forward pass
    lowercase__						:	Optional[int]         = model(
        UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		encoder_hidden_states=UpperCAmelCase__ ,		encoder_attention_mask=UpperCAmelCase__ ,		use_cache=UpperCAmelCase__ ,		)
    lowercase__						:	List[str]         = outputs.past_key_values

    # create hypothetical multiple next token and extent to next_input_ids
    lowercase__						:	List[str]         = ids_tensor((self.batch_size, 3) ,		config.vocab_size	)
    lowercase__						:	Dict         = ids_tensor((self.batch_size, 3) ,		vocab_size=2	)

    # append to next input_ids and
    lowercase__						:	Dict         = torch.cat([input_ids, next_tokens] ,		dim=-1	)
    lowercase__						:	List[Any]         = torch.cat([input_mask, next_mask] ,		dim=-1	)

    lowercase__						:	Optional[Any]         = model(
        UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		encoder_hidden_states=UpperCAmelCase__ ,		encoder_attention_mask=UpperCAmelCase__ ,		output_hidden_states=UpperCAmelCase__ ,		)['hidden_states'][0]
    lowercase__						:	Union[str, Any]         = model(
        UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		encoder_hidden_states=UpperCAmelCase__ ,		encoder_attention_mask=UpperCAmelCase__ ,		past_key_values=UpperCAmelCase__ ,		output_hidden_states=UpperCAmelCase__ ,		)['hidden_states'][0]

    # select random slice
    lowercase__						:	int         = ids_tensor((1,) ,		output_from_past.shape[-1]	).item()
    lowercase__						:	str         = output_from_no_past[:, -3:, random_slice_idx].detach()
    lowercase__						:	List[Any]         = output_from_past[:, :, random_slice_idx].detach()

    self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

    # test that outputs are equal for slice
    self.parent.assertTrue(torch.allclose(UpperCAmelCase__ ,		UpperCAmelCase__ ,		atol=1e-3	)	)



 def 						UpperCAmelCase    (	self   :   List[Any] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Optional[int]	):

    """simple docstring"""






    lowercase__						:	Dict         = RemBertForQuestionAnswering(config=UpperCAmelCase__	)
    model.to(UpperCAmelCase__	)
    model.eval()
    lowercase__						:	List[Any]         = model(
        UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		start_positions=UpperCAmelCase__ ,		end_positions=UpperCAmelCase__ ,		)
    self.parent.assertEqual(result.start_logits.shape ,		(self.batch_size, self.seq_length)	)
    self.parent.assertEqual(result.end_logits.shape ,		(self.batch_size, self.seq_length)	)



 def 						UpperCAmelCase    (	self   :   int ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   str ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   List[Any]	):

    """simple docstring"""






    lowercase__						:	Optional[Any]         = self.num_labels
    lowercase__						:	Union[str, Any]         = RemBertForSequenceClassification(UpperCAmelCase__	)
    model.to(UpperCAmelCase__	)
    model.eval()
    lowercase__						:	Any         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		labels=UpperCAmelCase__	)
    self.parent.assertEqual(result.logits.shape ,		(self.batch_size, self.num_labels)	)



 def 						UpperCAmelCase    (	self   :   Any ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Optional[Any] ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   List[str]	):

    """simple docstring"""






    lowercase__						:	Tuple         = self.num_labels
    lowercase__						:	Tuple         = RemBertForTokenClassification(config=UpperCAmelCase__	)
    model.to(UpperCAmelCase__	)
    model.eval()
    lowercase__						:	Optional[int]         = model(UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		labels=UpperCAmelCase__	)
    self.parent.assertEqual(result.logits.shape ,		(self.batch_size, self.seq_length, self.num_labels)	)



 def 						UpperCAmelCase    (	self   :   str ,		UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Any ,		UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   Tuple	):

    """simple docstring"""






    lowercase__						:	Any         = self.num_choices
    lowercase__						:	Dict         = RemBertForMultipleChoice(config=UpperCAmelCase__	)
    model.to(UpperCAmelCase__	)
    model.eval()
    lowercase__						:	Any         = input_ids.unsqueeze(1	).expand(-1 ,		self.num_choices ,		-1	).contiguous()
    lowercase__						:	List[Any]         = token_type_ids.unsqueeze(1	).expand(-1 ,		self.num_choices ,		-1	).contiguous()
    lowercase__						:	List[Any]         = input_mask.unsqueeze(1	).expand(-1 ,		self.num_choices ,		-1	).contiguous()
    lowercase__						:	str         = model(
        UpperCAmelCase__ ,		attention_mask=UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		labels=UpperCAmelCase__ ,		)
    self.parent.assertEqual(result.logits.shape ,		(self.batch_size, self.num_choices)	)







 def 						UpperCAmelCase    (	self   :   Optional[Any]	):

    """simple docstring"""






    lowercase__						:	Dict         = self.prepare_config_and_inputs()
    (
        lowercase__			
    )						:	int         = config_and_inputs
    lowercase__						:	Optional[Any]         = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
    return config, inputs_dict
@require_torch
class        __snake_case(					snake_case__     ,       snake_case__     ,       unittest.TestCase      ):
 lowercase :       int		      =		(
     (
         RemBertModel,
         RemBertForMaskedLM,
         RemBertForCausalLM,
         RemBertForMultipleChoice,
         RemBertForQuestionAnswering,
         RemBertForSequenceClassification,
         RemBertForTokenClassification,
     )
     if is_torch_available()
     else ()
 )
 lowercase :       Tuple		      =		(RemBertForCausalLM,) if is_torch_available() else ()
 lowercase :       List[Any]		      =		(
     {
         'feature-extraction': RemBertModel,
         'fill-mask': RemBertForMaskedLM,
         'question-answering': RemBertForQuestionAnswering,
         'text-classification': RemBertForSequenceClassification,
         'text-generation': RemBertForCausalLM,
         'token-classification': RemBertForTokenClassification,
         'zero-shot': RemBertForSequenceClassification,
     }
     if is_torch_available()
     else {}
 )



 def 						UpperCAmelCase    (	self   :   Dict	):

    """simple docstring"""






    lowercase__						:	Dict         = RemBertModelTester(self	)
    lowercase__						:	str         = ConfigTester(self ,		config_class=UpperCAmelCase__ ,		hidden_size=37	)



 def 						UpperCAmelCase    (	self   :   Optional[int]	):

    """simple docstring"""






    self.config_tester.run_common_tests()



 def 						UpperCAmelCase    (	self   :   List[Any]	):

    """simple docstring"""






    lowercase__						:	List[str]         = self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_model(*UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   Any	):

    """simple docstring"""






    lowercase__						:	Union[str, Any]         = self.model_tester.prepare_config_and_inputs()
    for type in ["absolute", "relative_key", "relative_key_query"]:
       lowercase__						:	Optional[Any]         = type
       self.model_tester.create_and_check_model(*UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   Dict	):

    """simple docstring"""






    lowercase__						:	Optional[Any]         = self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   int	):

    """simple docstring"""






    lowercase__						:	Optional[Any]         = self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   str	):

    """simple docstring"""






    lowercase__						:	Optional[Any]         = self.model_tester.prepare_config_and_inputs_for_decoder()
    self.model_tester.create_and_check_decoder_model_past_large_inputs(*UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   int	):

    """simple docstring"""






    lowercase__						:	List[Any]         = self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   str	):

    """simple docstring"""






    lowercase__						:	Optional[Any]         = self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   int	):

    """simple docstring"""






    lowercase__						:	Any         = self.model_tester.prepare_config_and_inputs()
    self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   List[str]	):

    """simple docstring"""






    lowercase__						:	Optional[Any]         = self.model_tester.prepare_config_and_inputs_for_decoder()
    self.model_tester.create_and_check_model_as_decoder(*UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   str	):

    """simple docstring"""






    (
        lowercase__			
    )						:	int         = self.model_tester.prepare_config_and_inputs_for_decoder()

    lowercase__						:	str         = None

    self.model_tester.create_and_check_model_as_decoder(
        UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		UpperCAmelCase__ ,		)







 @slow
 def 						UpperCAmelCase    (	self   :   Any	):

    """simple docstring"""






    for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       lowercase__						:	int         = RemBertModel.from_pretrained(UpperCAmelCase__	)
       self.assertIsNotNone(UpperCAmelCase__	)
@require_torch
class        __snake_case(					unittest.TestCase      ):



 @slow
 def 						UpperCAmelCase    (	self   :   Dict	):

    """simple docstring"""






    lowercase__						:	Dict         = RemBertModel.from_pretrained("""google/rembert"""	)
    lowercase__						:	List[Any]         = torch.tensor([[312, 56_498, 313, 2_125, 313]]	)
    lowercase__						:	Any         = torch.tensor([[0, 0, 0, 1, 1]]	)
    with torch.no_grad():
       lowercase__						:	Union[str, Any]         = model(UpperCAmelCase__ ,		token_type_ids=UpperCAmelCase__ ,		output_hidden_states=UpperCAmelCase__	)

    lowercase__						:	Union[str, Any]         = 1_152

    lowercase__						:	Optional[int]         = torch.Size((1, 5, hidden_size)	)
    self.assertEqual(output["""last_hidden_state"""].shape ,		UpperCAmelCase__	)

    lowercase__						:	Tuple         = torch.tensor(
        [
            [
                [0.0_754, -0.2_022, 0.1_904],
                [-0.3_354, -0.3_692, -0.4_791],
                [-0.2_314, -0.6_729, -0.0_749],
                [-0.0_396, -0.3_105, -0.4_234],
                [-0.1_571, -0.0_525, 0.5_353],
            ]
        ]	)

    # Running on the original tf implementation gives slightly different results here.
    # Not clear why this variations is present
    # TODO: Find reason for discrepancy
    # expected_original_implementation = [[
    #     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
    #     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
    #     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
    #     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
    #     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
    # ]]

    self.assertTrue(torch.allclose(output["""last_hidden_state"""][:, :, :3] ,		UpperCAmelCase__ ,		atol=1e-4	)	)










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a    =       {'configuration_nat': ['NAT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'NatConfig']}


try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a    =       [
     'NAT_PRETRAINED_MODEL_ARCHIVE_LIST',
     'NatForImageClassification',
     'NatModel',
     'NatPreTrainedModel',
     'NatBackbone',
 ]

if TYPE_CHECKING:
 from .configuration_nat import NAT_PRETRAINED_CONFIG_ARCHIVE_MAP, NatConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_nat import (
      NAT_PRETRAINED_MODEL_ARCHIVE_LIST,
      NatBackbone,
      NatForImageClassification,
      NatModel,
      NatPreTrainedModel,
  )

else:
 import sys

 a    =       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


