



import types
from typing import List, Optional, Tuple, Union

import torch
from transformers import CLIPTextModelWithProjection, CLIPTokenizer
from transformers.models.clip.modeling_clip import CLIPTextModelOutput

from diffusers.models import PriorTransformer
from diffusers.pipelines import DiffusionPipeline, StableDiffusionImageVariationPipeline
from diffusers.schedulers import UnCLIPScheduler
from diffusers.utils import logging, randn_tensor


_lowerCamelCase         =	logging.get_logger(__name__)  # pylint: disable=invalid-name






def 						_lowerCAmelCase    (					self :  Optional[Any]		,						__lowerCamelCase :  Optional[Any]		,						__lowerCamelCase :  Optional[Any]		,						__lowerCamelCase :  Optional[int]		,						__lowerCamelCase :  Optional[int]  ):



 """simple docstring"""





 __SCREAMING_SNAKE_CASE							:     int							=						image.to(device=__lowerCamelCase  )
 __SCREAMING_SNAKE_CASE							:     List[str]							=						image  # take image as image_embeddings
 __SCREAMING_SNAKE_CASE							:     int							=						image_embeddings.unsqueeze(1  )

 # duplicate image embeddings for each generation per prompt, using mps friendly method
 __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     Optional[int]							=						image_embeddings.shape
 __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						image_embeddings.repeat(1		,						__lowerCamelCase		,						1  )
 __SCREAMING_SNAKE_CASE							:     List[str]							=						image_embeddings.view(bs_embed * num_images_per_prompt		,						__lowerCamelCase		,						-1  )

 if do_classifier_free_guidance:
  __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						torch.zeros_like(__lowerCamelCase  )

  # For classifier free guidance, we need to do two forward passes.
  # Here we concatenate the unconditional and text embeddings into a single batch
  # to avoid doing two forward passes
  __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						torch.cat([uncond_embeddings, image_embeddings]  )

 return image_embeddings






class    _SCREAMING_SNAKE_CASE						(UpperCamelCase			):





     def __init__(							self			:				Optional[Any]  ,	UpperCamelCase			:				PriorTransformer  ,	UpperCamelCase			:				CLIPTokenizer  ,	UpperCamelCase			:				CLIPTextModelWithProjection  ,	UpperCamelCase			:				UnCLIPScheduler  ,	UpperCamelCase			:				Optional[dict] = None  ,	)->List[Any]:
      super().__init__()

      __SCREAMING_SNAKE_CASE							:     str							=						{"image_encoder": None} if decoder_pipe_kwargs is None else decoder_pipe_kwargs

      __SCREAMING_SNAKE_CASE							:     List[Any]							=						decoder_pipe_kwargs.get("torch_dtype"  ,	UpperCamelCase					) or prior.dtype

      __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						StableDiffusionImageVariationPipeline.from_pretrained(
          "lambdalabs/sd-image-variations-diffusers"  ,	**UpperCamelCase					)

      # replace `_encode_image` method
      __SCREAMING_SNAKE_CASE							:     List[str]							=						types.MethodType(_encode_image  ,	self.decoder_pipe					)

      self.register_modules(
          prior=UpperCamelCase  ,	tokenizer=UpperCamelCase  ,	text_encoder=UpperCamelCase  ,	prior_scheduler=UpperCamelCase  ,	)





     def 						__snake_case     (							self			:				Optional[Any]  ,	UpperCamelCase			:				Union[str, Any]  ,	UpperCamelCase			:				Optional[int]  ,	UpperCamelCase			:				Tuple  ,	UpperCamelCase			:				Tuple  ,	UpperCamelCase			:				Optional[Union[CLIPTextModelOutput, Tuple]] = None  ,	UpperCamelCase			:				Optional[torch.Tensor] = None  ,	)->Union[str, Any]:
      if text_model_output is None:
       __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						len(UpperCamelCase					) if isinstance(UpperCamelCase  ,	UpperCamelCase					) else 1
       # get prompt text embeddings
       __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						self.tokenizer(
           UpperCamelCase  ,	padding="max_length"  ,	max_length=self.tokenizer.model_max_length  ,	return_tensors="pt"  ,	)
       __SCREAMING_SNAKE_CASE							:     str							=						text_inputs.input_ids
       __SCREAMING_SNAKE_CASE							:     List[str]							=						text_inputs.attention_mask.bool().to(UpperCamelCase					)

       if text_input_ids.shape[-1] > self.tokenizer.model_max_length:
        __SCREAMING_SNAKE_CASE							:     List[str]							=						self.tokenizer.batch_decode(text_input_ids[:, self.tokenizer.model_max_length :]					)
        logger.warning(
            "The following part of your input was truncated because CLIP can only handle sequences up to"
            F""" {self.tokenizer.model_max_length} tokens: {removed_text}"""					)
        __SCREAMING_SNAKE_CASE							:     str							=						text_input_ids[:, : self.tokenizer.model_max_length]

       __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						self.text_encoder(text_input_ids.to(UpperCamelCase					)					)

       __SCREAMING_SNAKE_CASE							:     str							=						text_encoder_output.text_embeds
       __SCREAMING_SNAKE_CASE							:     str							=						text_encoder_output.last_hidden_state

      else:
       __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						text_model_output[0].shape[0]
       __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     List[str]							=						text_model_output[0], text_model_output[1]
       __SCREAMING_SNAKE_CASE							:     Any							=						text_attention_mask

      __SCREAMING_SNAKE_CASE							:     List[str]							=						text_embeddings.repeat_interleave(UpperCamelCase  ,	dim=0					)
      __SCREAMING_SNAKE_CASE							:     Any							=						text_encoder_hidden_states.repeat_interleave(UpperCamelCase  ,	dim=0					)
      __SCREAMING_SNAKE_CASE							:     List[Any]							=						text_mask.repeat_interleave(UpperCamelCase  ,	dim=0					)

      if do_classifier_free_guidance:
       __SCREAMING_SNAKE_CASE							:     Tuple							=						[""] * batch_size

       __SCREAMING_SNAKE_CASE							:     Any							=						self.tokenizer(
           UpperCamelCase  ,	padding="max_length"  ,	max_length=self.tokenizer.model_max_length  ,	truncation=UpperCamelCase  ,	return_tensors="pt"  ,	)
       __SCREAMING_SNAKE_CASE							:     Dict							=						uncond_input.attention_mask.bool().to(UpperCamelCase					)
       __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						self.text_encoder(uncond_input.input_ids.to(UpperCamelCase					)					)

       __SCREAMING_SNAKE_CASE							:     Any							=						uncond_embeddings_text_encoder_output.text_embeds
       __SCREAMING_SNAKE_CASE							:     Tuple							=						uncond_embeddings_text_encoder_output.last_hidden_state

       # duplicate unconditional embeddings for each generation per prompt, using mps friendly method

       __SCREAMING_SNAKE_CASE							:     Tuple							=						uncond_embeddings.shape[1]
       __SCREAMING_SNAKE_CASE							:     Tuple							=						uncond_embeddings.repeat(1  ,	UpperCamelCase					)
       __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						uncond_embeddings.view(batch_size * num_images_per_prompt  ,	UpperCamelCase					)

       __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						uncond_text_encoder_hidden_states.shape[1]
       __SCREAMING_SNAKE_CASE							:     str							=						uncond_text_encoder_hidden_states.repeat(1  ,	UpperCamelCase  ,	1					)
       __SCREAMING_SNAKE_CASE							:     List[Any]							=						uncond_text_encoder_hidden_states.view(
           batch_size * num_images_per_prompt  ,	UpperCamelCase  ,	-1					)
       __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						uncond_text_mask.repeat_interleave(UpperCamelCase  ,	dim=0					)

       # done duplicates

       # For classifier free guidance, we need to do two forward passes.
       # Here we concatenate the unconditional and text embeddings into a single batch
       # to avoid doing two forward passes
       __SCREAMING_SNAKE_CASE							:     Optional[int]							=						torch.cat([uncond_embeddings, text_embeddings]					)
       __SCREAMING_SNAKE_CASE							:     List[str]							=						torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]					)

       __SCREAMING_SNAKE_CASE							:     Tuple							=						torch.cat([uncond_text_mask, text_mask]					)

      return text_embeddings, text_encoder_hidden_states, text_mask





     @property
     def 						__snake_case     (							self			:				Optional[int]					)->Any:
      if self.device != torch.device("meta"					) or not hasattr(self.prior  ,	"_hf_hook"					):
       return self.device
      for module in self.prior.modules():
       if (
           hasattr(UpperCamelCase  ,	"_hf_hook"					)
           and hasattr(module._hf_hook  ,	"execution_device"					)
           and module._hf_hook.execution_device is not None
       ):
        return torch.device(module._hf_hook.execution_device					)
      return self.device





     def 						__snake_case     (							self			:				Optional[Any]  ,	UpperCamelCase			:				Tuple  ,	UpperCamelCase			:				Any  ,	UpperCamelCase			:				Dict  ,	UpperCamelCase			:				Any  ,	UpperCamelCase			:				Optional[Any]  ,	UpperCamelCase			:				List[str]					)->Dict:
      if latents is None:
       __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						randn_tensor(UpperCamelCase  ,	generator=UpperCamelCase  ,	device=UpperCamelCase  ,	dtype=UpperCamelCase					)
      else:
       if latents.shape != shape:
        raise ValueError(F"""Unexpected latents shape, got {latents.shape}, expected {shape}"""					)
       __SCREAMING_SNAKE_CASE							:     Optional[int]							=						latents.to(UpperCamelCase					)

      __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						latents * scheduler.init_noise_sigma
      return latents





     def 						__snake_case     (							self			:				List[Any]  ,	UpperCamelCase			:				Optional[Union[str, torch.device]] = None					)->List[str]:
      self.decoder_pipe.to(UpperCamelCase					)
      super().to(UpperCamelCase					)






     @torch.no_grad()
     def __call__(							self			:				Any  ,	UpperCamelCase			:				Optional[Union[str, List[str]]] = None  ,	UpperCamelCase			:				Optional[int] = None  ,	UpperCamelCase			:				Optional[int] = None  ,	UpperCamelCase			:				int = 1  ,	UpperCamelCase			:				int = 2_5  ,	UpperCamelCase			:				Optional[torch.Generator] = None  ,	UpperCamelCase			:				Optional[torch.FloatTensor] = None  ,	UpperCamelCase			:				Optional[Union[CLIPTextModelOutput, Tuple]] = None  ,	UpperCamelCase			:				Optional[torch.Tensor] = None  ,	UpperCamelCase			:				float = 4.0  ,	UpperCamelCase			:				float = 8.0  ,	UpperCamelCase			:				int = 5_0  ,	UpperCamelCase			:				Optional[int] = 1  ,	UpperCamelCase			:				float = 0.0  ,	UpperCamelCase			:				Optional[str] = "pil"  ,	UpperCamelCase			:				bool = True  ,	)->int:
      if prompt is not None:
       if isinstance(UpperCamelCase  ,	UpperCamelCase					):
        __SCREAMING_SNAKE_CASE							:     List[Any]							=						1
       elif isinstance(UpperCamelCase  ,	UpperCamelCase					):
        __SCREAMING_SNAKE_CASE							:     str							=						len(UpperCamelCase					)
       else:
        raise ValueError(F"""`prompt` has to be of type `str` or `list` but is {type(UpperCamelCase					)}"""					)
      else:
       __SCREAMING_SNAKE_CASE							:     int							=						text_model_output[0].shape[0]

      __SCREAMING_SNAKE_CASE							:     Tuple							=						self._execution_device

      __SCREAMING_SNAKE_CASE							:     int							=						batch_size * num_images_per_prompt

      __SCREAMING_SNAKE_CASE							:     List[Any]							=						prior_guidance_scale > 1.0 or decoder_guidance_scale > 1.0

      __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     int							=						self._encode_prompt(
          UpperCamelCase  ,	UpperCamelCase  ,	UpperCamelCase  ,	UpperCamelCase  ,	UpperCamelCase  ,	UpperCamelCase					)

      # prior

      self.prior_scheduler.set_timesteps(UpperCamelCase  ,	device=UpperCamelCase					)
      __SCREAMING_SNAKE_CASE							:     Optional[int]							=						self.prior_scheduler.timesteps

      __SCREAMING_SNAKE_CASE							:     Optional[Any]							=						self.prior.config.embedding_dim

      __SCREAMING_SNAKE_CASE							:     int							=						self.prepare_latents(
          (batch_size, embedding_dim)  ,	text_embeddings.dtype  ,	UpperCamelCase  ,	UpperCamelCase  ,	UpperCamelCase  ,	self.prior_scheduler  ,	)

      for i, t in enumerate(self.progress_bar(UpperCamelCase					)					):
       # expand the latents if we are doing classifier free guidance
       __SCREAMING_SNAKE_CASE							:     str							=						torch.cat([prior_latents] * 2					) if do_classifier_free_guidance else prior_latents

       __SCREAMING_SNAKE_CASE							:     str							=						self.prior(
           UpperCamelCase  ,	timestep=UpperCamelCase  ,	proj_embedding=UpperCamelCase  ,	encoder_hidden_states=UpperCamelCase  ,	attention_mask=UpperCamelCase  ,	).predicted_image_embedding

       if do_classifier_free_guidance:
        __SCREAMING_SNAKE_CASE,		__SCREAMING_SNAKE_CASE							:     Optional[Any]							=						predicted_image_embedding.chunk(2					)
        __SCREAMING_SNAKE_CASE							:     List[str]							=						predicted_image_embedding_uncond + prior_guidance_scale * (
            predicted_image_embedding_text - predicted_image_embedding_uncond
        )

       if i + 1 == prior_timesteps_tensor.shape[0]:
        __SCREAMING_SNAKE_CASE							:     Optional[int]							=						None
       else:
        __SCREAMING_SNAKE_CASE							:     Tuple							=						prior_timesteps_tensor[i + 1]

       __SCREAMING_SNAKE_CASE							:     Any							=						self.prior_scheduler.step(
           UpperCamelCase  ,	timestep=UpperCamelCase  ,	sample=UpperCamelCase  ,	generator=UpperCamelCase  ,	prev_timestep=UpperCamelCase  ,	).prev_sample

      __SCREAMING_SNAKE_CASE							:     Tuple							=						self.prior.post_process_latents(UpperCamelCase					)

      __SCREAMING_SNAKE_CASE							:     Union[str, Any]							=						prior_latents

      __SCREAMING_SNAKE_CASE							:     Any							=						self.decoder_pipe(
          image=UpperCamelCase  ,	height=UpperCamelCase  ,	width=UpperCamelCase  ,	num_inference_steps=UpperCamelCase  ,	guidance_scale=UpperCamelCase  ,	generator=UpperCamelCase  ,	output_type=UpperCamelCase  ,	return_dict=UpperCamelCase  ,	num_images_per_prompt=UpperCamelCase  ,	eta=UpperCamelCase  ,	)
      return output







import unittest

from transformers import YosoConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        YosoForMaskedLM,
        YosoForMultipleChoice,
        YosoForQuestionAnswering,
        YosoForSequenceClassification,
        YosoForTokenClassification,
        YosoModel,
    )
    from transformers.models.yoso.modeling_yoso import YOSO_PRETRAINED_MODEL_ARCHIVE_LIST




class 						__snake_case						:





      '''simple docstring'''






      def __init__(       self  :					List[Any]     ,	_UpperCamelCase  :					Union[str, Any]     ,	_UpperCamelCase  :					Tuple=13     ,	_UpperCamelCase  :					Union[str, Any]=7     ,	_UpperCamelCase  :					Union[str, Any]=True     ,	_UpperCamelCase  :					Dict=True     ,	_UpperCamelCase  :					List[Any]=True     ,	_UpperCamelCase  :					Optional[Any]=True     ,	_UpperCamelCase  :					Optional[int]=99     ,	_UpperCamelCase  :					List[Any]=32     ,	_UpperCamelCase  :					Dict=5     ,	_UpperCamelCase  :					Dict=4     ,	_UpperCamelCase  :					Optional[int]=37     ,	_UpperCamelCase  :					List[Any]="gelu"     ,	_UpperCamelCase  :					int=0.1     ,	_UpperCamelCase  :					str=0.1     ,	_UpperCamelCase  :					int=512     ,	_UpperCamelCase  :					Optional[Any]=16     ,	_UpperCamelCase  :					Tuple=2     ,	_UpperCamelCase  :					Any=0.0_2     ,	_UpperCamelCase  :					Any=3     ,	_UpperCamelCase  :					Union[str, Any]=4     ,	_UpperCamelCase  :					List[Any]=None     ,	)					->List[Any]:


          """simple docstring"""


          _lowerCamelCase  :		List[str]			    =    parent
          _lowerCamelCase  :		List[Any]			    =    batch_size
          _lowerCamelCase  :		int			    =    seq_length
          _lowerCamelCase  :		List[Any]			    =    is_training
          _lowerCamelCase  :		List[Any]			    =    use_input_mask
          _lowerCamelCase  :		Tuple			    =    use_token_type_ids
          _lowerCamelCase  :		int			    =    use_labels
          _lowerCamelCase  :		int			    =    vocab_size
          _lowerCamelCase  :		List[str]			    =    hidden_size
          _lowerCamelCase  :		List[Any]			    =    num_hidden_layers
          _lowerCamelCase  :		str			    =    num_attention_heads
          _lowerCamelCase  :		List[str]			    =    intermediate_size
          _lowerCamelCase  :		List[Any]			    =    hidden_act
          _lowerCamelCase  :		Optional[int]			    =    hidden_dropout_prob
          _lowerCamelCase  :		Optional[int]			    =    attention_probs_dropout_prob
          _lowerCamelCase  :		Tuple			    =    max_position_embeddings
          _lowerCamelCase  :		List[str]			    =    type_vocab_size
          _lowerCamelCase  :		Dict			    =    type_sequence_label_size
          _lowerCamelCase  :		str			    =    initializer_range
          _lowerCamelCase  :		Tuple			    =    num_labels
          _lowerCamelCase  :		Union[str, Any]			    =    num_choices
          _lowerCamelCase  :		str			    =    scope



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Union[str, Any])					->str:


          """simple docstring"""


          _lowerCamelCase  :		Optional[int]			    =    ids_tensor([self.batch_size, self.seq_length]     ,	self.vocab_size)

          _lowerCamelCase  :		Tuple			    =    None
          if self.use_input_mask:
              _lowerCamelCase  :		Dict			    =    random_attention_mask([self.batch_size, self.seq_length])

          _lowerCamelCase  :		str			    =    None
          if self.use_token_type_ids:
              _lowerCamelCase  :		Optional[Any]			    =    ids_tensor([self.batch_size, self.seq_length]     ,	self.type_vocab_size)

          _lowerCamelCase  :		List[str]			    =    None
          _lowerCamelCase  :		Optional[int]			    =    None
          _lowerCamelCase  :		Dict			    =    None
          if self.use_labels:
              _lowerCamelCase  :		List[str]			    =    ids_tensor([self.batch_size]     ,	self.type_sequence_label_size)
              _lowerCamelCase  :		List[str]			    =    ids_tensor([self.batch_size, self.seq_length]     ,	self.num_labels)
              _lowerCamelCase  :		Any			    =    ids_tensor([self.batch_size]     ,	self.num_choices)

          _lowerCamelCase  :		Optional[Any]			    =    self.get_config()

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Optional[int])					->str:


          """simple docstring"""


          return YosoConfig(
              vocab_size=self.vocab_size     ,	hidden_size=self.hidden_size     ,	num_hidden_layers=self.num_hidden_layers     ,	num_attention_heads=self.num_attention_heads     ,	intermediate_size=self.intermediate_size     ,	hidden_act=self.hidden_act     ,	hidden_dropout_prob=self.hidden_dropout_prob     ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,	max_position_embeddings=self.max_position_embeddings     ,	type_vocab_size=self.type_vocab_size     ,	is_decoder=_UpperCamelCase     ,	initializer_range=self.initializer_range     ,	)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Union[str, Any])					->Optional[Any]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.get_config()
          _lowerCamelCase  :		List[str]			    =    300
          return config



      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[Any])					->Optional[Any]:


          """simple docstring"""


          (
              (
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				
          )  :		Dict			    =    self.prepare_config_and_inputs()

          _lowerCamelCase  :		int			    =    True
          _lowerCamelCase  :		List[Any]			    =    floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
          _lowerCamelCase  :		Any			    =    ids_tensor([self.batch_size, self.seq_length]     ,	vocab_size=2)

          return (
              config,
              input_ids,
              token_type_ids,
              input_mask,
              sequence_labels,
              token_labels,
              choice_labels,
              encoder_hidden_states,
              encoder_attention_mask,
          )



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Any     ,	_UpperCamelCase  :					Tuple     ,	_UpperCamelCase  :					Optional[Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Optional[int]     ,	_UpperCamelCase  :					Any     ,	_UpperCamelCase  :					Optional[int])					->str:


          """simple docstring"""


          _lowerCamelCase  :		str			    =    YosoModel(config=_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		Union[str, Any]			    =    model(_UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase)
          _lowerCamelCase  :		Optional[Any]			    =    model(_UpperCamelCase     ,	token_type_ids=_UpperCamelCase)
          _lowerCamelCase  :		Optional[int]			    =    model(_UpperCamelCase)
          self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, self.seq_length, self.hidden_size))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					str     ,	_UpperCamelCase  :					Union[str, Any]     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					Any     ,	_UpperCamelCase  :					int     ,	_UpperCamelCase  :					Optional[Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					List[Any]     ,	_UpperCamelCase  :					Optional[int]     ,	)					->List[str]:


          """simple docstring"""


          _lowerCamelCase  :		Optional[Any]			    =    True
          _lowerCamelCase  :		str			    =    YosoModel(_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		Optional[int]			    =    model(
              _UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	encoder_hidden_states=_UpperCamelCase     ,	encoder_attention_mask=_UpperCamelCase     ,	)
          _lowerCamelCase  :		Union[str, Any]			    =    model(
              _UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	encoder_hidden_states=_UpperCamelCase     ,	)
          _lowerCamelCase  :		List[str]			    =    model(_UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase)
          self.parent.assertEqual(result.last_hidden_state.shape     ,	(self.batch_size, self.seq_length, self.hidden_size))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Any     ,	_UpperCamelCase  :					int     ,	_UpperCamelCase  :					Optional[int]     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Tuple     ,	_UpperCamelCase  :					Optional[Any]     ,	_UpperCamelCase  :					Optional[Any]     ,	_UpperCamelCase  :					str)					->Tuple:


          """simple docstring"""


          _lowerCamelCase  :		List[Any]			    =    YosoForMaskedLM(config=_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		Tuple			    =    model(_UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	labels=_UpperCamelCase)
          self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.vocab_size))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Union[str, Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					Any     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Optional[int]     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Union[str, Any])					->Any:


          """simple docstring"""


          _lowerCamelCase  :		Tuple			    =    YosoForQuestionAnswering(config=_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		int			    =    model(
              _UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	start_positions=_UpperCamelCase     ,	end_positions=_UpperCamelCase     ,	)
          self.parent.assertEqual(result.start_logits.shape     ,	(self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape     ,	(self.batch_size, self.seq_length))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					Any     ,	_UpperCamelCase  :					Union[str, Any]     ,	_UpperCamelCase  :					Union[str, Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					str)					->Optional[int]:


          """simple docstring"""


          _lowerCamelCase  :		List[Any]			    =    self.num_labels
          _lowerCamelCase  :		Tuple			    =    YosoForSequenceClassification(_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		Optional[Any]			    =    model(_UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	labels=_UpperCamelCase)
          self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_labels))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[Any]     ,	_UpperCamelCase  :					List[str]     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					Any     ,	_UpperCamelCase  :					List[str]     ,	_UpperCamelCase  :					Union[str, Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					str)					->List[str]:


          """simple docstring"""


          _lowerCamelCase  :		Tuple			    =    self.num_labels
          _lowerCamelCase  :		Union[str, Any]			    =    YosoForTokenClassification(config=_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		str			    =    model(_UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	labels=_UpperCamelCase)
          self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.seq_length, self.num_labels))



      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[Any]     ,	_UpperCamelCase  :					Tuple     ,	_UpperCamelCase  :					List[str]     ,	_UpperCamelCase  :					Optional[Any]     ,	_UpperCamelCase  :					str     ,	_UpperCamelCase  :					Dict     ,	_UpperCamelCase  :					int     ,	_UpperCamelCase  :					Tuple)					->str:


          """simple docstring"""


          _lowerCamelCase  :		Any			    =    self.num_choices
          _lowerCamelCase  :		Tuple			    =    YosoForMultipleChoice(config=_UpperCamelCase)
          model.to(_UpperCamelCase)
          model.eval()
          _lowerCamelCase  :		Optional[int]			    =    input_ids.unsqueeze(1).expand(-1     ,	self.num_choices     ,	-1).contiguous()
          _lowerCamelCase  :		Union[str, Any]			    =    token_type_ids.unsqueeze(1).expand(-1     ,	self.num_choices     ,	-1).contiguous()
          _lowerCamelCase  :		Optional[Any]			    =    input_mask.unsqueeze(1).expand(-1     ,	self.num_choices     ,	-1).contiguous()
          _lowerCamelCase  :		str			    =    model(
              _UpperCamelCase     ,	attention_mask=_UpperCamelCase     ,	token_type_ids=_UpperCamelCase     ,	labels=_UpperCamelCase     ,	)
          self.parent.assertEqual(result.logits.shape     ,	(self.batch_size, self.num_choices))




      def 						_SCREAMING_SNAKE_CASE     (       self  :					Tuple)					->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase  :		Dict			    =    self.prepare_config_and_inputs()
          (
              (
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				(
              _lowerCamelCase
          )    ,				
          )  :		Tuple			    =    config_and_inputs
          _lowerCamelCase  :		Union[str, Any]			    =    {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
          return config, inputs_dict





@require_torch
class 						__snake_case						(			__lowerCAmelCase			,	__lowerCAmelCase			,	unittest.TestCase		):





      '''simple docstring'''






      _snake_case     		=							(
          (
              YosoModel,
              YosoForMaskedLM,
              YosoForMultipleChoice,
              YosoForQuestionAnswering,
              YosoForSequenceClassification,
              YosoForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      _snake_case     		=							False
      _snake_case     		=							False
      _snake_case     		=							False

      _snake_case     		=							()
      _snake_case     		=							(
          {
              'feature-extraction': YosoModel,
              'fill-mask': YosoForMaskedLM,
              'question-answering': YosoForQuestionAnswering,
              'text-classification': YosoForSequenceClassification,
              'token-classification': YosoForTokenClassification,
              'zero-shot': YosoForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Optional[int])					->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    YosoModelTester(self)
          _lowerCamelCase  :		Optional[int]			    =    ConfigTester(self     ,	config_class=_UpperCamelCase     ,	hidden_size=37)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Optional[Any])					->Optional[Any]:


          """simple docstring"""


          self.config_tester.run_common_tests()



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Optional[int])					->List[str]:


          """simple docstring"""


          _lowerCamelCase  :		Optional[Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Any)					->Tuple:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
              _lowerCamelCase  :		List[str]			    =    type
              self.model_tester.create_and_check_model(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Optional[Any])					->Optional[int]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Dict)					->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[Any])					->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					Tuple)					->Union[str, Any]:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*_UpperCamelCase)



      def 						_SCREAMING_SNAKE_CASE     (       self  :					str)					->Any:


          """simple docstring"""


          _lowerCamelCase  :		List[Any]			    =    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*_UpperCamelCase)



      @slow
      def 						_SCREAMING_SNAKE_CASE     (       self  :					str)					->List[str]:


          """simple docstring"""


          for model_name in YOSO_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              _lowerCamelCase  :		Tuple			    =    YosoModel.from_pretrained(_UpperCamelCase)
              self.assertIsNotNone(_UpperCamelCase)




      def 						_SCREAMING_SNAKE_CASE     (       self  :					List[str])					->Union[str, Any]:


          """simple docstring"""


          return





@require_torch
class 						__snake_case						(			unittest.TestCase		):





      '''simple docstring'''






      @slow
      def 						_SCREAMING_SNAKE_CASE     (       self  :					Dict)					->Dict:


          """simple docstring"""


          _lowerCamelCase  :		List[Any]			    =    YosoModel.from_pretrained("""uw-madison/yoso-4096""")
          _lowerCamelCase  :		int			    =    torch.tensor([[0, 1, 2, 3, 4, 5]])

          with torch.no_grad():
              _lowerCamelCase  :		Any			    =    model(_UpperCamelCase)[0]

          _lowerCamelCase  :		List[Any]			    =    torch.Size((1, 6, 768))
          self.assertEqual(output.shape     ,	_UpperCamelCase)

          _lowerCamelCase  :		int			    =    torch.tensor(
              [[[-0.0_6_1_1, 0.1_2_4_2, 0.0_8_4_0], [0.0_2_8_0, -0.0_0_4_8, 0.1_1_2_5], [0.0_1_0_6, 0.0_2_2_6, 0.0_7_5_1]]])

          self.assertTrue(torch.allclose(output[:, :3, :3]     ,	_UpperCamelCase     ,	atol=1E-4))



      @slow
      def 						_SCREAMING_SNAKE_CASE     (       self  :					Union[str, Any])					->str:


          """simple docstring"""


          _lowerCamelCase  :		Union[str, Any]			    =    YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096""")
          _lowerCamelCase  :		Tuple			    =    torch.tensor([[0, 1, 2, 3, 4, 5]])

          with torch.no_grad():
              _lowerCamelCase  :		Optional[Any]			    =    model(_UpperCamelCase)[0]

          _lowerCamelCase  :		Union[str, Any]			    =    5_0265

          _lowerCamelCase  :		Any			    =    torch.Size((1, 6, vocab_size))
          self.assertEqual(output.shape     ,	_UpperCamelCase)

          _lowerCamelCase  :		List[str]			    =    torch.tensor(
              [[[-2.1_3_1_3, -3.7_2_8_5, -2.2_4_0_7], [-2.7_0_4_7, -3.3_3_1_4, -2.6_4_0_8], [0.0_6_2_9, -2.5_1_6_6, -0.3_3_5_6]]])

          self.assertTrue(torch.allclose(output[:, :3, :3]     ,	_UpperCamelCase     ,	atol=1E-4))




      @slow
      def 						_SCREAMING_SNAKE_CASE     (       self  :					str)					->Tuple:


          """simple docstring"""


          _lowerCamelCase  :		Tuple			    =    YosoForMaskedLM.from_pretrained("""uw-madison/yoso-4096""")
          _lowerCamelCase  :		Any			    =    torch.arange(4096).unsqueeze(0)

          with torch.no_grad():
              _lowerCamelCase  :		Optional[Any]			    =    model(_UpperCamelCase)[0]

          _lowerCamelCase  :		Dict			    =    5_0265

          _lowerCamelCase  :		str			    =    torch.Size((1, 4096, vocab_size))
          self.assertEqual(output.shape     ,	_UpperCamelCase)

          _lowerCamelCase  :		int			    =    torch.tensor(
              [[[-2.3_9_1_4, -4.3_7_4_2, -5.0_9_5_6], [-4.0_9_8_8, -4.2_3_8_4, -7.0_4_0_6], [-3.1_4_2_7, -3.7_1_9_2, -6.6_8_0_0]]])

          self.assertTrue(torch.allclose(output[:, :3, :3]     ,	_UpperCamelCase     ,	atol=1E-4))




