
import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    RemBertConfig,
				    RemBertForCausalLM,
				    RemBertForMaskedLM,
				    RemBertForMultipleChoice,
				    RemBertForQuestionAnswering,
				    RemBertForSequenceClassification,
				    RemBertForTokenClassification,
				    RemBertModel,
				)
				from transformers.models.rembert.modeling_rembert import REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST







class 							UpperCAmelCase	:

					def __init__(			self      :					Union[str, Any] ,      lowerCAmelCase      :					Tuple ,      lowerCAmelCase      :					Union[str, Any]=13 ,      lowerCAmelCase      :					Tuple=7 ,      lowerCAmelCase      :					List[Any]=True ,      lowerCAmelCase      :					Optional[Any]=True ,      lowerCAmelCase      :					Optional[int]=True ,      lowerCAmelCase      :					Dict=True ,      lowerCAmelCase      :					Optional[Any]=99 ,      lowerCAmelCase      :					str=32 ,      lowerCAmelCase      :					Tuple=18 ,      lowerCAmelCase      :					Any=43 ,      lowerCAmelCase      :					int=5 ,      lowerCAmelCase      :					Tuple=4 ,      lowerCAmelCase      :					Union[str, Any]=37 ,      lowerCAmelCase      :					Any="gelu" ,      lowerCAmelCase      :					int=0.1 ,      lowerCAmelCase      :					Optional[Any]=0.1 ,      lowerCAmelCase      :					Optional[int]=512 ,      lowerCAmelCase      :					Any=16 ,      lowerCAmelCase      :					Tuple=2 ,      lowerCAmelCase      :					str=0.02 ,      lowerCAmelCase      :					Optional[Any]=3 ,      lowerCAmelCase      :					int=4 ,      lowerCAmelCase      :					Union[str, Any]=None ,      ):
											lowercase       :    Tuple											=  parent
											lowercase       :    str											=  batch_size
											lowercase       :    int											=  seq_length
											lowercase       :    Tuple											=  is_training
											lowercase       :    List[str]											=  use_input_mask
											lowercase       :    Optional[Any]											=  use_token_type_ids
											lowercase       :    Union[str, Any]											=  use_labels
											lowercase       :    Optional[Any]											=  vocab_size
											lowercase       :    List[Any]											=  hidden_size
											lowercase       :    Optional[Any]											=  input_embedding_size
											lowercase       :    Any											=  output_embedding_size
											lowercase       :    Tuple											=  num_hidden_layers
											lowercase       :    Optional[int]											=  num_attention_heads
											lowercase       :    Optional[int]											=  intermediate_size
											lowercase       :    Optional[Any]											=  hidden_act
											lowercase       :    Optional[int]											=  hidden_dropout_prob
											lowercase       :    str											=  attention_probs_dropout_prob
											lowercase       :    int											=  max_position_embeddings
											lowercase       :    List[str]											=  type_vocab_size
											lowercase       :    List[str]											=  type_sequence_label_size
											lowercase       :    str											=  initializer_range
											lowercase       :    List[str]											=  num_labels
											lowercase       :    int											=  num_choices
											lowercase       :    Optional[int]											=  scope

					def        _lowerCAmelCase	(			self      :					Any					):
											lowercase       :    Dict											=  ids_tensor([self.batch_size, self.seq_length] ,      self.vocab_size					)

											lowercase       :    Tuple											=  None
											if self.use_input_mask:
																	lowercase       :    str											=  random_attention_mask([self.batch_size, self.seq_length]					)

											lowercase       :    List[Any]											=  None
											if self.use_token_type_ids:
																	lowercase       :    List[str]											=  ids_tensor([self.batch_size, self.seq_length] ,      self.type_vocab_size					)

											lowercase       :    Optional[Any]											=  None
											lowercase       :    Optional[Any]											=  None
											lowercase       :    Union[str, Any]											=  None
											if self.use_labels:
																	lowercase       :    int											=  ids_tensor([self.batch_size] ,      self.type_sequence_label_size					)
																	lowercase       :    Dict											=  ids_tensor([self.batch_size, self.seq_length] ,      self.num_labels					)
																	lowercase       :    Dict											=  ids_tensor([self.batch_size] ,      self.num_choices					)

											lowercase       :    Tuple											=  RemBertConfig(
											    vocab_size=self.vocab_size ,      hidden_size=self.hidden_size ,      input_embedding_size=self.input_embedding_size ,      output_embedding_size=self.output_embedding_size ,      num_hidden_layers=self.num_hidden_layers ,      num_attention_heads=self.num_attention_heads ,      intermediate_size=self.intermediate_size ,      hidden_act=self.hidden_act ,      hidden_dropout_prob=self.hidden_dropout_prob ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob ,      max_position_embeddings=self.max_position_embeddings ,      type_vocab_size=self.type_vocab_size ,      is_decoder=lowerCAmelCase ,      initializer_range=self.initializer_range ,      )

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

					def        _lowerCAmelCase	(			self      :					Optional[int]					):
											(
											    (
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				
											)       :    Dict											=  self.prepare_config_and_inputs()

											lowercase       :    int											=  True
											lowercase       :    List[str]											=  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]					)
											lowercase       :    Optional[Any]											=  ids_tensor([self.batch_size, self.seq_length] ,      vocab_size=2					)

											return (
											    config,
											    input_ids,
											    token_type_ids,
											    input_mask,
											    sequence_labels,
											    token_labels,
											    choice_labels,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)

					def        _lowerCAmelCase	(			self      :					int ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					Tuple ,      lowerCAmelCase      :					Optional[Any] ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					List[str]					):
											lowercase       :    Dict											=  RemBertModel(config=lowerCAmelCase					)
											model.to(lowerCAmelCase					)
											model.eval()
											lowercase       :    int											=  model(lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase					)
											lowercase       :    int											=  model(lowerCAmelCase ,      token_type_ids=lowerCAmelCase					)
											lowercase       :    Optional[int]											=  model(lowerCAmelCase					)
											self.parent.assertEqual(result.last_hidden_state.shape ,      (self.batch_size, self.seq_length, self.hidden_size)					)

					def        _lowerCAmelCase	(			self      :					Any ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					str ,      lowerCAmelCase      :					Dict ,      lowerCAmelCase      :					Tuple ,      ):
											lowercase       :    Optional[int]											=  True
											lowercase       :    List[str]											=  RemBertModel(lowerCAmelCase					)
											model.to(lowerCAmelCase					)
											model.eval()
											lowercase       :    Any											=  model(
											    lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      encoder_hidden_states=lowerCAmelCase ,      encoder_attention_mask=lowerCAmelCase ,      )
											lowercase       :    Union[str, Any]											=  model(
											    lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      encoder_hidden_states=lowerCAmelCase ,      )
											lowercase       :    Union[str, Any]											=  model(lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase					)
											self.parent.assertEqual(result.last_hidden_state.shape ,      (self.batch_size, self.seq_length, self.hidden_size)					)

					def        _lowerCAmelCase	(			self      :					Dict ,      lowerCAmelCase      :					Optional[Any] ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					List[str] ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					Tuple ,      lowerCAmelCase      :					List[Any] ,      lowerCAmelCase      :					List[str] ,      ):
											lowercase       :    Optional[int]											=  RemBertForCausalLM(config=lowerCAmelCase					)
											model.to(lowerCAmelCase					)
											model.eval()
											lowercase       :    List[Any]											=  model(lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      labels=lowerCAmelCase					)
											self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.seq_length, self.vocab_size)					)

					def        _lowerCAmelCase	(			self      :					int ,      lowerCAmelCase      :					Dict ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					Optional[Any] ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					Optional[int]					):
											lowercase       :    List[Any]											=  RemBertForMaskedLM(config=lowerCAmelCase					)
											model.to(lowerCAmelCase					)
											model.eval()
											lowercase       :    Dict											=  model(lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      labels=lowerCAmelCase					)
											self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.seq_length, self.vocab_size)					)

					def        _lowerCAmelCase	(			self      :					List[str] ,      lowerCAmelCase      :					Optional[Any] ,      lowerCAmelCase      :					Optional[Any] ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					Optional[Any] ,      lowerCAmelCase      :					str ,      lowerCAmelCase      :					List[Any] ,      lowerCAmelCase      :					int ,      lowerCAmelCase      :					Tuple ,      ):
											lowercase       :    Tuple											=  True
											lowercase       :    Tuple											=  True
											lowercase       :    Dict											=  RemBertForCausalLM(config=lowerCAmelCase					)
											model.to(lowerCAmelCase					)
											model.eval()

											# first forward pass
											lowercase       :    Union[str, Any]											=  model(
											    lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      encoder_hidden_states=lowerCAmelCase ,      encoder_attention_mask=lowerCAmelCase ,      use_cache=lowerCAmelCase ,      )
											lowercase       :    Optional[Any]											=  outputs.past_key_values

											# create hypothetical multiple next token and extent to next_input_ids
											lowercase       :    Union[str, Any]											=  ids_tensor((self.batch_size, 3) ,      config.vocab_size					)
											lowercase       :    List[Any]											=  ids_tensor((self.batch_size, 3) ,      vocab_size=2					)

											# append to next input_ids and
											lowercase       :    List[str]											=  torch.cat([input_ids, next_tokens] ,      dim=-1					)
											lowercase       :    int											=  torch.cat([input_mask, next_mask] ,      dim=-1					)

											lowercase       :    str											=  model(
											    lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      encoder_hidden_states=lowerCAmelCase ,      encoder_attention_mask=lowerCAmelCase ,      output_hidden_states=lowerCAmelCase ,      )['''hidden_states'''][0]
											lowercase       :    str											=  model(
											    lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      encoder_hidden_states=lowerCAmelCase ,      encoder_attention_mask=lowerCAmelCase ,      past_key_values=lowerCAmelCase ,      output_hidden_states=lowerCAmelCase ,      )['''hidden_states'''][0]

											# select random slice
											lowercase       :    List[str]											=  ids_tensor((1,) ,      output_from_past.shape[-1]					).item()
											lowercase       :    Optional[int]											=  output_from_no_past[:, -3:, random_slice_idx].detach()
											lowercase       :    Union[str, Any]											=  output_from_past[:, :, random_slice_idx].detach()

											self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]					)

											# test that outputs are equal for slice
											self.parent.assertTrue(torch.allclose(lowerCAmelCase ,      lowerCAmelCase ,      atol=1E-3					)					)

					def        _lowerCAmelCase	(			self      :					Dict ,      lowerCAmelCase      :					List[Any] ,      lowerCAmelCase      :					Tuple ,      lowerCAmelCase      :					Tuple ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					Dict ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					Optional[int]					):
											lowercase       :    List[Any]											=  RemBertForQuestionAnswering(config=lowerCAmelCase					)
											model.to(lowerCAmelCase					)
											model.eval()
											lowercase       :    int											=  model(
											    lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      start_positions=lowerCAmelCase ,      end_positions=lowerCAmelCase ,      )
											self.parent.assertEqual(result.start_logits.shape ,      (self.batch_size, self.seq_length)					)
											self.parent.assertEqual(result.end_logits.shape ,      (self.batch_size, self.seq_length)					)

					def        _lowerCAmelCase	(			self      :					Optional[int] ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					Tuple ,      lowerCAmelCase      :					List[str] ,      lowerCAmelCase      :					str ,      lowerCAmelCase      :					List[Any] ,      lowerCAmelCase      :					Optional[Any] ,      lowerCAmelCase      :					Dict					):
											lowercase       :    Union[str, Any]											=  self.num_labels
											lowercase       :    Optional[Any]											=  RemBertForSequenceClassification(lowerCAmelCase					)
											model.to(lowerCAmelCase					)
											model.eval()
											lowercase       :    Tuple											=  model(lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      labels=lowerCAmelCase					)
											self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.num_labels)					)

					def        _lowerCAmelCase	(			self      :					Union[str, Any] ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					Optional[int] ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					Dict ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					Union[str, Any] ,      lowerCAmelCase      :					Dict					):
											lowercase       :    int											=  self.num_labels
											lowercase       :    Optional[int]											=  RemBertForTokenClassification(config=lowerCAmelCase					)
											model.to(lowerCAmelCase					)
											model.eval()
											lowercase       :    Optional[int]											=  model(lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      labels=lowerCAmelCase					)
											self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.seq_length, self.num_labels)					)

					def        _lowerCAmelCase	(			self      :					Optional[Any] ,      lowerCAmelCase      :					Tuple ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					Dict ,      lowerCAmelCase      :					List[str] ,      lowerCAmelCase      :					Any ,      lowerCAmelCase      :					Union[str, Any]					):
											lowercase       :    Optional[Any]											=  self.num_choices
											lowercase       :    Optional[int]											=  RemBertForMultipleChoice(config=lowerCAmelCase					)
											model.to(lowerCAmelCase					)
											model.eval()
											lowercase       :    str											=  input_ids.unsqueeze(1					).expand(-1 ,      self.num_choices ,      -1					).contiguous()
											lowercase       :    List[str]											=  token_type_ids.unsqueeze(1					).expand(-1 ,      self.num_choices ,      -1					).contiguous()
											lowercase       :    str											=  input_mask.unsqueeze(1					).expand(-1 ,      self.num_choices ,      -1					).contiguous()
											lowercase       :    int											=  model(
											    lowerCAmelCase ,      attention_mask=lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      labels=lowerCAmelCase ,      )
											self.parent.assertEqual(result.logits.shape ,      (self.batch_size, self.num_choices)					)




					def        _lowerCAmelCase	(			self      :					List[Any]					):
											lowercase       :    int											=  self.prepare_config_and_inputs()
											(
											    (
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				
											)       :    Optional[int]											=  config_and_inputs
											lowercase       :    Tuple											=  {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
											return config, inputs_dict







@require_torch
class 							UpperCAmelCase	(   __lowerCamelCase						, __lowerCamelCase						, unittest.TestCase					):
					a__: str			  =				(
					    (
					        RemBertModel,
					        RemBertForMaskedLM,
					        RemBertForCausalLM,
					        RemBertForMultipleChoice,
					        RemBertForQuestionAnswering,
					        RemBertForSequenceClassification,
					        RemBertForTokenClassification,
					    )
					    if is_torch_available()
					    else ()
					)
					a__: str			  =				(RemBertForCausalLM,) if is_torch_available() else ()
					a__: List[Any]			  =				(
					    {
					        """feature-extraction""": RemBertModel,
					        """fill-mask""": RemBertForMaskedLM,
					        """question-answering""": RemBertForQuestionAnswering,
					        """text-classification""": RemBertForSequenceClassification,
					        """text-generation""": RemBertForCausalLM,
					        """token-classification""": RemBertForTokenClassification,
					        """zero-shot""": RemBertForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)

					def        _lowerCAmelCase	(			self      :					Union[str, Any]					):
											lowercase       :    Optional[Any]											=  RemBertModelTester(self					)
											lowercase       :    str											=  ConfigTester(self ,      config_class=lowerCAmelCase ,      hidden_size=37					)

					def        _lowerCAmelCase	(			self      :					str					):
											self.config_tester.run_common_tests()

					def        _lowerCAmelCase	(			self      :					Optional[int]					):
											lowercase       :    List[str]											=  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*lowerCAmelCase					)

					def        _lowerCAmelCase	(			self      :					List[str]					):
											lowercase       :    List[Any]											=  self.model_tester.prepare_config_and_inputs()
											for type in ["absolute", "relative_key", "relative_key_query"]:
																	lowercase       :    Dict											=  type
																	self.model_tester.create_and_check_model(*lowerCAmelCase					)

					def        _lowerCAmelCase	(			self      :					Dict					):
											lowercase       :    Optional[Any]											=  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase					)

					def        _lowerCAmelCase	(			self      :					List[Any]					):
											lowercase       :    Optional[Any]											=  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase					)

					def        _lowerCAmelCase	(			self      :					str					):
											lowercase       :    Optional[Any]											=  self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_decoder_model_past_large_inputs(*lowerCAmelCase					)

					def        _lowerCAmelCase	(			self      :					Optional[int]					):
											lowercase       :    Any											=  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase					)

					def        _lowerCAmelCase	(			self      :					List[Any]					):
											lowercase       :    Tuple											=  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase					)

					def        _lowerCAmelCase	(			self      :					List[Any]					):
											lowercase       :    Union[str, Any]											=  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase					)

					def        _lowerCAmelCase	(			self      :					int					):
											lowercase       :    Dict											=  self.model_tester.prepare_config_and_inputs_for_decoder()
											self.model_tester.create_and_check_model_as_decoder(*lowerCAmelCase					)

					def        _lowerCAmelCase	(			self      :					Tuple					):
											# This regression test was failing with PyTorch < 1.3
											(
											    (
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				(
											    lowercase
											)				,				
											)       :    List[str]											=  self.model_tester.prepare_config_and_inputs_for_decoder()

											lowercase       :    List[Any]											=  None

											self.model_tester.create_and_check_model_as_decoder(
											    lowerCAmelCase ,      lowerCAmelCase ,      lowerCAmelCase ,      lowerCAmelCase ,      lowerCAmelCase ,      lowerCAmelCase ,      lowerCAmelCase ,      lowerCAmelCase ,      lowerCAmelCase ,      )




					@slow
					def        _lowerCAmelCase	(			self      :					Optional[int]					):
											for model_name in REMBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	lowercase       :    int											=  RemBertModel.from_pretrained(lowerCAmelCase					)
																	self.assertIsNotNone(lowerCAmelCase					)







@require_torch
class 							UpperCAmelCase	(   unittest.TestCase					):

					@slow
					def        _lowerCAmelCase	(			self      :					Tuple					):
											# Test exact values at the last hidden layer
											lowercase       :    Optional[int]											=  RemBertModel.from_pretrained('''google/rembert'''					)
											lowercase       :    Optional[Any]											=  torch.tensor([[312, 5_6498, 313, 2125, 313]]					)
											lowercase       :    Dict											=  torch.tensor([[0, 0, 0, 1, 1]]					)
											with torch.no_grad():
																	lowercase       :    Optional[int]											=  model(lowerCAmelCase ,      token_type_ids=lowerCAmelCase ,      output_hidden_states=lowerCAmelCase					)

											lowercase       :    Optional[int]											=  1152

											lowercase       :    Dict											=  torch.Size((1, 5, hidden_size)					)
											self.assertEqual(output['''last_hidden_state'''].shape ,      lowerCAmelCase					)

											lowercase       :    int											=  torch.tensor(
											    [
											        [
											            [0.07_54, -0.20_22, 0.19_04],
											            [-0.33_54, -0.36_92, -0.47_91],
											            [-0.23_14, -0.67_29, -0.07_49],
											            [-0.03_96, -0.31_05, -0.42_34],
											            [-0.15_71, -0.05_25, 0.53_53],
											        ]
											    ]					)

											# Running on the original tf implementation gives slightly different results here.
											# Not clear why this variations is present
											# TODO: Find reason for discrepancy
											# expected_original_implementation = [[
											#     [0.07630594074726105, -0.20146065950393677, 0.19107051193714142],
											#     [-0.3405614495277405, -0.36971670389175415, -0.4808273911476135],
											#     [-0.22587086260318756, -0.6656315922737122, -0.07844287157058716],
											#     [-0.04145475849509239, -0.3077218234539032, -0.42316967248916626],
											#     [-0.15887849032878876, -0.054529931396245956, 0.5356100797653198]
											# ]]

											self.assertTrue(torch.allclose(output['''last_hidden_state'''][:, :, :3] ,      lowerCAmelCase ,      atol=1E-4					)					)




snake_case__  							=							"""
# Diffusers installation
! pip install diffusers transformers datasets accelerate
# To install from source instead of the last release, comment the command above and uncomment the following one.
# ! pip install git+https://github.com/huggingface/diffusers.git
"""

snake_case__  							=							[{"""type""": """code""", """content""": INSTALL_CONTENT}]


