



'''simple docstring'''

from dataclasses import dataclass
from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..models.embeddings import ImagePositionalEmbeddings
from ..utils import BaseOutput, deprecate
from .attention import BasicTransformerBlock
from .embeddings import PatchEmbed
from .modeling_utils import ModelMixin

@dataclass
class 			__magic_name__		(					_UpperCamelCase     ):
							lowerCAmelCase :					torch.FloatTensor

class 			__magic_name__		(					_UpperCamelCase    ,		_UpperCamelCase     ):







							@register_to_config
							def __init__(							self      : List[str]			,_UpperCAmelCase      : int = 16			,_UpperCAmelCase      : int = 88			,_UpperCAmelCase      : Optional[int] = None			,_UpperCAmelCase      : Optional[int] = None			,_UpperCAmelCase      : int = 1			,_UpperCAmelCase      : float = 0.0			,_UpperCAmelCase      : int = 32			,_UpperCAmelCase      : Optional[int] = None			,_UpperCAmelCase      : bool = False			,_UpperCAmelCase      : Optional[int] = None			,_UpperCAmelCase      : Optional[int] = None			,_UpperCAmelCase      : Optional[int] = None			,_UpperCAmelCase      : str = "geglu"			,_UpperCAmelCase      : Optional[int] = None			,_UpperCAmelCase      : bool = False			,_UpperCAmelCase      : bool = False			,_UpperCAmelCase      : bool = False			,_UpperCAmelCase      : str = "layer_norm"			,_UpperCAmelCase      : bool = True			,):
									super().__init__()
									_a  :   int				 =							use_linear_projection
									_a  :   Optional[Any]				 =							num_attention_heads
									_a  :   List[str]				 =							attention_head_dim
									_a  :   Tuple				 =							num_attention_heads * attention_head_dim

									# 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)`
									# Define whether input is continuous or discrete depending on configuration
									_a  :   str				 =							(in_channels is not None) and (patch_size is None)
									_a  :   List[str]				 =							num_vector_embeds is not None
									_a  :   Optional[int]				 =							in_channels is not None and patch_size is not None

									if norm_type == "layer_norm" and num_embeds_ada_norm is not None:
											_a  :   Union[str, Any]				 =							(
											    F"""The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or"""
											    ' incorrectly set to `\'layer_norm\'`.Make sure to set `norm_type` to `\'ada_norm\'` in the config.'
											    ' Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect'
											    ' results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it'
											    ' would be very nice if you could open a Pull request for the `transformer/config.json` file'
											)
											deprecate('norm_type!=num_embeds_ada_norm'			,'1.0.0'			,_UpperCAmelCase			,standard_warn=_UpperCAmelCase	)
											_a  :   str				 =							'ada_norm'

									if self.is_input_continuous and self.is_input_vectorized:
											raise ValueError(
											    F"""Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make"""
											    ' sure that either `in_channels` or `num_vector_embeds` is None.'	)
									elif self.is_input_vectorized and self.is_input_patches:
											raise ValueError(
											    F"""Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make"""
											    ' sure that either `num_vector_embeds` or `num_patches` is None.'	)
									elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches:
											raise ValueError(
											    F"""Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:"""
											    F""" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None."""	)

									# 2. Define input layers
									if self.is_input_continuous:
											_a  :   Optional[int]				 =							in_channels

											_a  :   Union[str, Any]				 =							torch.nn.GroupNorm(num_groups=_UpperCAmelCase			,num_channels=_UpperCAmelCase			,eps=1E-6			,affine=_UpperCAmelCase	)
											if use_linear_projection:
													_a  :   str				 =							nn.Linear(_UpperCAmelCase			,_UpperCAmelCase	)
											else:
													_a  :   Any				 =							nn.Convad(_UpperCAmelCase			,_UpperCAmelCase			,kernel_size=1			,stride=1			,padding=0	)
									elif self.is_input_vectorized:
											assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size"
											assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed"

											_a  :   Union[str, Any]				 =							sample_size
											_a  :   int				 =							sample_size
											_a  :   Any				 =							num_vector_embeds
											_a  :   Optional[Any]				 =							self.height * self.width

											_a  :   List[str]				 =							ImagePositionalEmbeddings(
											    num_embed=_UpperCAmelCase			,embed_dim=_UpperCAmelCase			,height=self.height			,width=self.width	)
									elif self.is_input_patches:
											assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size"

											_a  :   Union[str, Any]				 =							sample_size
											_a  :   Tuple				 =							sample_size

											_a  :   Union[str, Any]				 =							patch_size
											_a  :   Any				 =							PatchEmbed(
											    height=_UpperCAmelCase			,width=_UpperCAmelCase			,patch_size=_UpperCAmelCase			,in_channels=_UpperCAmelCase			,embed_dim=_UpperCAmelCase			,)

									# 3. Define transformers blocks
									_a  :   Optional[Any]				 =							nn.ModuleList(
									    [
									        BasicTransformerBlock(
									            _UpperCAmelCase			,_UpperCAmelCase			,_UpperCAmelCase			,dropout=_UpperCAmelCase			,cross_attention_dim=_UpperCAmelCase			,activation_fn=_UpperCAmelCase			,num_embeds_ada_norm=_UpperCAmelCase			,attention_bias=_UpperCAmelCase			,only_cross_attention=_UpperCAmelCase			,upcast_attention=_UpperCAmelCase			,norm_type=_UpperCAmelCase			,norm_elementwise_affine=_UpperCAmelCase			,)
									        for d in range(_UpperCAmelCase	)
									    ]	)

									# 4. Define output layers
									_a  :   str				 =							in_channels if out_channels is None else out_channels
									if self.is_input_continuous:
											# TODO: should use out_channels for continuous projections
											if use_linear_projection:
													_a  :   List[Any]				 =							nn.Linear(_UpperCAmelCase			,_UpperCAmelCase	)
											else:
													_a  :   Dict				 =							nn.Convad(_UpperCAmelCase			,_UpperCAmelCase			,kernel_size=1			,stride=1			,padding=0	)
									elif self.is_input_vectorized:
											_a  :   str				 =							nn.LayerNorm(_UpperCAmelCase	)
											_a  :   int				 =							nn.Linear(_UpperCAmelCase			,self.num_vector_embeds - 1	)
									elif self.is_input_patches:
											_a  :   Optional[Any]				 =							nn.LayerNorm(_UpperCAmelCase			,elementwise_affine=_UpperCAmelCase			,eps=1E-6	)
											_a  :   int				 =							nn.Linear(_UpperCAmelCase			,2 * inner_dim	)
											_a  :   List[Any]				 =							nn.Linear(_UpperCAmelCase			,patch_size * patch_size * self.out_channels	)







							def 							__lowercase    (							self      : Union[str, Any]			,_UpperCAmelCase      : torch.Tensor			,_UpperCAmelCase      : Optional[torch.Tensor] = None			,_UpperCAmelCase      : Optional[torch.LongTensor] = None			,_UpperCAmelCase      : Optional[torch.LongTensor] = None			,_UpperCAmelCase      : Dict[str, Any] = None			,_UpperCAmelCase      : Optional[torch.Tensor] = None			,_UpperCAmelCase      : Optional[torch.Tensor] = None			,_UpperCAmelCase      : bool = True			,):
									# ensure attention_mask is a bias, and give it a singleton query_tokens dimension.
									#   we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward.
									#   we can tell by counting dims; if ndim == 2: it's a mask rather than a bias.
									# expects mask of shape:
									#   [batch, key_tokens]
									# adds singleton query_tokens dimension:
									#   [batch,                    1, key_tokens]
									# this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes:
									#   [batch,  heads, query_tokens, key_tokens] (e.g. torch sdp attn)
									#   [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn)
									if attention_mask is not None and attention_mask.ndim == 2:
											# assume that mask is expressed as:
											#   (1 = keep,      0 = discard)
											# convert mask into a bias that can be added to attention scores:
											#       (keep = +0,     discard = -10000.0)
											_a  :   List[str]				 =							(1 - attention_mask.to(hidden_states.dtype	)) * -1_00_00.0
											_a  :   Optional[Any]				 =							attention_mask.unsqueeze(1	)

									# convert encoder_attention_mask to a bias the same way we do for attention_mask
									if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2:
											_a  :   Union[str, Any]				 =							(1 - encoder_attention_mask.to(hidden_states.dtype	)) * -1_00_00.0
											_a  :   str				 =							encoder_attention_mask.unsqueeze(1	)

									# 1. Input
									if self.is_input_continuous:
											_a ,      _a ,      _a ,      _a  :   List[Any]				 =							hidden_states.shape
											_a  :   int				 =							hidden_states

											_a  :   Optional[Any]				 =							self.norm(_UpperCAmelCase	)
											if not self.use_linear_projection:
													_a  :   List[str]				 =							self.proj_in(_UpperCAmelCase	)
													_a  :   int				 =							hidden_states.shape[1]
													_a  :   Any				 =							hidden_states.permute(0			,2			,3			,1	).reshape(_UpperCAmelCase			,height * width			,_UpperCAmelCase	)
											else:
													_a  :   Dict				 =							hidden_states.shape[1]
													_a  :   Any				 =							hidden_states.permute(0			,2			,3			,1	).reshape(_UpperCAmelCase			,height * width			,_UpperCAmelCase	)
													_a  :   Optional[int]				 =							self.proj_in(_UpperCAmelCase	)
									elif self.is_input_vectorized:
											_a  :   Optional[Any]				 =							self.latent_image_embedding(_UpperCAmelCase	)
									elif self.is_input_patches:
											_a  :   Tuple				 =							self.pos_embed(_UpperCAmelCase	)

									# 2. Blocks
									for block in self.transformer_blocks:
											_a  :   List[Any]				 =							block(
											    _UpperCAmelCase			,attention_mask=_UpperCAmelCase			,encoder_hidden_states=_UpperCAmelCase			,encoder_attention_mask=_UpperCAmelCase			,timestep=_UpperCAmelCase			,cross_attention_kwargs=_UpperCAmelCase			,class_labels=_UpperCAmelCase			,)

									# 3. Output
									if self.is_input_continuous:
											if not self.use_linear_projection:
													_a  :   List[str]				 =							hidden_states.reshape(_UpperCAmelCase			,_UpperCAmelCase			,_UpperCAmelCase			,_UpperCAmelCase	).permute(0			,3			,1			,2	).contiguous()
													_a  :   Optional[int]				 =							self.proj_out(_UpperCAmelCase	)
											else:
													_a  :   Optional[int]				 =							self.proj_out(_UpperCAmelCase	)
													_a  :   Optional[Any]				 =							hidden_states.reshape(_UpperCAmelCase			,_UpperCAmelCase			,_UpperCAmelCase			,_UpperCAmelCase	).permute(0			,3			,1			,2	).contiguous()

											_a  :   str				 =							hidden_states + residual
									elif self.is_input_vectorized:
											_a  :   Optional[int]				 =							self.norm_out(_UpperCAmelCase	)
											_a  :   Optional[Any]				 =							self.out(_UpperCAmelCase	)
											# (batch, self.num_vector_embeds - 1, self.num_latent_pixels)
											_a  :   Optional[int]				 =							logits.permute(0			,2			,1	)

											# log(p(x_0))
											_a  :   List[Any]				 =							F.log_softmax(logits.double()			,dim=1	).float()
									elif self.is_input_patches:
											# TODO: cleanup!
											_a  :   str				 =							self.transformer_blocks[0].norma.emb(
											    _UpperCAmelCase			,_UpperCAmelCase			,hidden_dtype=hidden_states.dtype	)
											_a ,      _a  :   Optional[int]				 =							self.proj_out_a(F.silu(_UpperCAmelCase	)	).chunk(2			,dim=1	)
											_a  :   int				 =							self.norm_out(_UpperCAmelCase	) * (1 + scale[:, None]) + shift[:, None]
											_a  :   int				 =							self.proj_out_a(_UpperCAmelCase	)

											# unpatchify
											_a  :   str				 =							int(hidden_states.shape[1] ** 0.5	)
											_a  :   Any				 =							hidden_states.reshape(
											    shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels)	)
											_a  :   Any				 =							torch.einsum('nhwpqc->nchpwq'			,_UpperCAmelCase	)
											_a  :   str				 =							hidden_states.reshape(
											    shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size)	)

									if not return_dict:
											return (output,)

									return TransformeraDModelOutput(sample=_UpperCAmelCase	)









'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__lowerCAmelCase		    =     {'''configuration_t5''': ['''T5_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''T5Config''', '''T5OnnxConfig''']}

try:
							if not is_sentencepiece_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase		    =     ['''T5Tokenizer''']

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase		    =     ['''T5TokenizerFast''']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase		    =     [
							    '''T5_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''T5EncoderModel''',
							    '''T5ForConditionalGeneration''',
							    '''T5Model''',
							    '''T5PreTrainedModel''',
							    '''load_tf_weights_in_t5''',
							    '''T5ForQuestionAnswering''',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase		    =     [
							    '''TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''TFT5EncoderModel''',
							    '''TFT5ForConditionalGeneration''',
							    '''TFT5Model''',
							    '''TFT5PreTrainedModel''',
							]

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase		    =     [
							    '''FlaxT5EncoderModel''',
							    '''FlaxT5ForConditionalGeneration''',
							    '''FlaxT5Model''',
							    '''FlaxT5PreTrainedModel''',
							]


if TYPE_CHECKING:
							from .configuration_ta import T5_PRETRAINED_CONFIG_ARCHIVE_MAP, TaConfig, TaOnnxConfig

							try:
														if not is_sentencepiece_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_ta import TaTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_ta_fast import TaTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_ta import (
														    T5_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TaEncoderModel,
														    TaForConditionalGeneration,
														    TaForQuestionAnswering,
														    TaModel,
														    TaPreTrainedModel,
														    load_tf_weights_in_ta,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_ta import (
														    TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFTaEncoderModel,
														    TFTaForConditionalGeneration,
														    TFTaModel,
														    TFTaPreTrainedModel,
														)

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_ta import (
														    FlaxTaEncoderModel,
														    FlaxTaForConditionalGeneration,
														    FlaxTaModel,
														    FlaxTaPreTrainedModel,
														)


else:
							import sys

							__lowerCAmelCase		    =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




