





"""simple docstring"""




import math
from typing import Optional

import numpy as np
import torch
from torch import nn

from .activations import get_activation





def 		UpperCAmelCase__						(	_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase = False					,   _UpperCAmelCase = 1					,   _UpperCAmelCase = 1					,   _UpperCAmelCase = 10000					,   ):






  """simple docstring"""





  assert len(timesteps.shape   ) == 1, "Timesteps should be a 1d-array"

  A_    :			Dict									=    embedding_dim // 2
  A_    :			Optional[Any]									=    -math.log(_UpperCAmelCase   ) * torch.arange(
      start=0					,   end=_UpperCAmelCase					,   dtype=torch.floataa					,   device=timesteps.device   )
  A_    :			List[str]									=    exponent / (half_dim - downscale_freq_shift)

  A_    :			str									=    torch.exp(_UpperCAmelCase   )
  A_    :			str									=    timesteps[:, None].float() * emb[None, :]

  # scale embeddings
  A_    :			Union[str, Any]									=    scale * emb

  # concat sine and cosine embeddings
  A_    :			List[Any]									=    torch.cat([torch.sin(_UpperCAmelCase   ), torch.cos(_UpperCAmelCase   )]					,   dim=-1   )

  # flip sine and cosine embeddings
  if flip_sin_to_cos:
    A_    :			Any									=    torch.cat([emb[:, half_dim:], emb[:, :half_dim]]					,   dim=-1   )

  # zero pad
  if embedding_dim % 2 == 1:
    A_    :			str									=    torch.nn.functional.pad(_UpperCAmelCase					,   (0, 1, 0, 0)   )
  return emb





def 		UpperCAmelCase__						(	_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase=False					,   _UpperCAmelCase=0   ):






  """simple docstring"""





  A_    :			Union[str, Any]									=    np.arange(_UpperCAmelCase					,   dtype=np.floataa   )
  A_    :			Any									=    np.arange(_UpperCAmelCase					,   dtype=np.floataa   )
  A_    :			Tuple									=    np.meshgrid(_UpperCAmelCase					,   _UpperCAmelCase   )  # here w goes first
  A_    :			List[Any]									=    np.stack(_UpperCAmelCase					,   axis=0   )

  A_    :			Optional[int]									=    grid.reshape([2, 1, grid_size, grid_size]   )
  A_    :			Dict									=    get_ad_sincos_pos_embed_from_grid(_UpperCAmelCase					,   _UpperCAmelCase   )
  if cls_token and extra_tokens > 0:
    A_    :			List[str]									=    np.concatenate([np.zeros([extra_tokens, embed_dim]   ), pos_embed]					,   axis=0   )
  return pos_embed





def 		UpperCAmelCase__						(	_UpperCAmelCase					,   _UpperCAmelCase   ):






  """simple docstring"""





  if embed_dim % 2 != 0:
    raise ValueError('embed_dim must be divisible by 2'   )

  # use half of dimensions to encode grid_h
  A_    :			Union[str, Any]									=    get_ad_sincos_pos_embed_from_grid(embed_dim // 2					,   grid[0]   )  # (H*W, D/2)
  A_    :			str									=    get_ad_sincos_pos_embed_from_grid(embed_dim // 2					,   grid[1]   )  # (H*W, D/2)

  A_    :			List[Any]									=    np.concatenate([emb_h, emb_w]					,   axis=1   )  # (H*W, D)
  return emb





def 		UpperCAmelCase__						(	_UpperCAmelCase					,   _UpperCAmelCase   ):






  """simple docstring"""





  if embed_dim % 2 != 0:
    raise ValueError('embed_dim must be divisible by 2'   )

  A_    :			Tuple									=    np.arange(embed_dim // 2					,   dtype=np.floataa   )
  omega /= embed_dim / 2.0
  A_    :			Optional[Any]									=    1.0 / 10000**omega  # (D/2,)

  A_    :			Tuple									=    pos.reshape(-1   )  # (M,)
  A_    :			List[Any]									=    np.einsum('m,d->md'					,   _UpperCAmelCase					,   _UpperCAmelCase   )  # (M, D/2), outer product

  A_    :			str									=    np.sin(_UpperCAmelCase   )  # (M, D/2)
  A_    :			Union[str, Any]									=    np.cos(_UpperCAmelCase   )  # (M, D/2)

  A_    :			int									=    np.concatenate([emb_sin, emb_cos]					,   axis=1   )  # (M, D)
  return emb

class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_=2_2_4   , snake_case_=2_2_4   , snake_case_=1_6   , snake_case_=3   , snake_case_=7_6_8   , snake_case_=False   , snake_case_=True   , snake_case_=True   , ):

        """simple docstring"""

        super().__init__()

        A_    :			List[Any]									=    (height // patch_size) * (width // patch_size)
        A_    :			Optional[Any]									=    flatten
        A_    :			Dict									=    layer_norm

        A_    :			str									=    nn.Convad(
            snake_case_   , snake_case_   , kernel_size=(patch_size, patch_size)   , stride=snake_case_   , bias=snake_case_    )
        if layer_norm:
          A_    :			str									=    nn.LayerNorm(snake_case_   , elementwise_affine=snake_case_   , eps=1E-6    )
        else:
          A_    :			int									=    None

        A_    :			List[str]									=    get_ad_sincos_pos_embed(snake_case_   , int(num_patches**0.5    )    )
        self.register_buffer('pos_embed'   , torch.from_numpy(snake_case_    ).float().unsqueeze(0    )   , persistent=snake_case_    )

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_    :			List[str]									=    self.proj(snake_case_    )
        if self.flatten:
          A_    :			Dict									=    latent.flatten(2    ).transpose(1   , 2    )  # BCHW -> BNC
        if self.layer_norm:
          A_    :			List[Any]									=    self.norm(snake_case_    )
        return latent + self.pos_embed


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_   , snake_case_ = "silu"   , snake_case_ = None   , snake_case_ = None   , snake_case_=None   , ):

        """simple docstring"""

        super().__init__()

        A_    :			str									=    nn.Linear(snake_case_   , snake_case_    )

        if cond_proj_dim is not None:
          A_    :			Tuple									=    nn.Linear(snake_case_   , snake_case_   , bias=snake_case_    )
        else:
          A_    :			Any									=    None

        A_    :			List[Any]									=    get_activation(snake_case_    )

        if out_dim is not None:
          A_    :			Dict									=    out_dim
        else:
          A_    :			Union[str, Any]									=    time_embed_dim
        A_    :			Dict									=    nn.Linear(snake_case_   , snake_case_    )

        if post_act_fn is None:
          A_    :			Union[str, Any]									=    None
        else:
          A_    :			List[str]									=    get_activation(snake_case_    )

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_=None    ):

        """simple docstring"""

        if condition is not None:
          A_    :			List[Any]									=    sample + self.cond_proj(snake_case_    )
        A_    :			Optional[int]									=    self.linear_a(snake_case_    )

        if self.act is not None:
          A_    :			Dict									=    self.act(snake_case_    )

        A_    :			Optional[int]									=    self.linear_a(snake_case_    )

        if self.post_act is not None:
          A_    :			Optional[int]									=    self.post_act(snake_case_    )
        return sample


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_   , snake_case_    ):

        """simple docstring"""

        super().__init__()
        A_    :			List[Any]									=    num_channels
        A_    :			Dict									=    flip_sin_to_cos
        A_    :			Optional[Any]									=    downscale_freq_shift

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_    :			Union[str, Any]									=    get_timestep_embedding(
            snake_case_   , self.num_channels   , flip_sin_to_cos=self.flip_sin_to_cos   , downscale_freq_shift=self.downscale_freq_shift   , )
        return t_emb


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_ = 2_5_6   , snake_case_ = 1.0   , snake_case_=True   , snake_case_=True   , snake_case_=False    ):

        """simple docstring"""

        super().__init__()
        A_    :			List[Any]									=    nn.Parameter(torch.randn(snake_case_    ) * scale   , requires_grad=snake_case_    )
        A_    :			List[str]									=    log
        A_    :			Optional[Any]									=    flip_sin_to_cos

        if set_W_to_weight:
          # to delete later
          A_    :			List[Any]									=    nn.Parameter(torch.randn(snake_case_    ) * scale   , requires_grad=snake_case_    )

          A_    :			Any									=    self.W

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        if self.log:
          A_    :			List[Any]									=    torch.log(snake_case_    )

        A_    :			List[Any]									=    x[:, None] * self.weight[None, :] * 2 * np.pi

        if self.flip_sin_to_cos:
          A_    :			List[str]									=    torch.cat([torch.cos(snake_case_    ), torch.sin(snake_case_    )]   , dim=-1    )
        else:
          A_    :			Optional[Any]									=    torch.cat([torch.sin(snake_case_    ), torch.cos(snake_case_    )]   , dim=-1    )
        return out


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_   , snake_case_   , snake_case_   , ):

        """simple docstring"""

        super().__init__()

        A_    :			Optional[int]									=    height
        A_    :			Union[str, Any]									=    width
        A_    :			Optional[Any]									=    num_embed
        A_    :			Dict									=    embed_dim

        A_    :			Union[str, Any]									=    nn.Embedding(self.num_embed   , snake_case_    )
        A_    :			str									=    nn.Embedding(self.height   , snake_case_    )
        A_    :			Tuple									=    nn.Embedding(self.width   , snake_case_    )

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_    :			List[str]									=    self.emb(snake_case_    )

        A_    :			Optional[Any]									=    self.height_emb(torch.arange(self.height   , device=index.device    ).view(1   , self.height    )    )

        # 1 x H x D -> 1 x H x 1 x D
        A_    :			str									=    height_emb.unsqueeze(2    )

        A_    :			Optional[Any]									=    self.width_emb(torch.arange(self.width   , device=index.device    ).view(1   , self.width    )    )

        # 1 x W x D -> 1 x 1 x W x D
        A_    :			List[str]									=    width_emb.unsqueeze(1    )

        A_    :			List[Any]									=    height_emb + width_emb

        # 1 x H x W x D -> 1 x L xD
        A_    :			Optional[int]									=    pos_emb.view(1   , self.height * self.width   , -1    )

        A_    :			Optional[int]									=    emb + pos_emb[:, : emb.shape[1], :]

        return emb


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_   , snake_case_    ):

        """simple docstring"""

        super().__init__()
        A_    :			List[Any]									=    dropout_prob > 0
        A_    :			List[str]									=    nn.Embedding(num_classes + use_cfg_embedding   , snake_case_    )
        A_    :			int									=    num_classes
        A_    :			Optional[int]									=    dropout_prob

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_=None    ):

        """simple docstring"""

        if force_drop_ids is None:
          A_    :			Optional[Any]									=    torch.rand(labels.shape[0]   , device=labels.device    ) < self.dropout_prob
        else:
          A_    :			Optional[Any]									=    torch.tensor(force_drop_ids == 1    )
        A_    :			List[Any]									=    torch.where(snake_case_   , self.num_classes   , snake_case_    )
        return labels

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_=None    ):

        """simple docstring"""

        A_    :			Optional[int]									=    self.dropout_prob > 0
        if (self.training and use_dropout) or (force_drop_ids is not None):
          A_    :			Optional[int]									=    self.token_drop(snake_case_   , snake_case_    )
        A_    :			Optional[Any]									=    self.embedding_table(snake_case_    )
        return embeddings


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_ = 1_0_2_4   , snake_case_ = 7_6_8   , snake_case_ = 7_6_8   , snake_case_ = 1_0   , ):

        """simple docstring"""

        super().__init__()

        A_    :			Optional[int]									=    num_image_text_embeds
        A_    :			str									=    nn.Linear(snake_case_   , self.num_image_text_embeds * cross_attention_dim    )
        A_    :			Optional[Any]									=    nn.Linear(snake_case_   , snake_case_    )

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_    ):

        """simple docstring"""

        A_    :			int									=    text_embeds.shape[0]

        # image
        A_    :			int									=    self.image_embeds(snake_case_    )
        A_    :			Tuple									=    image_text_embeds.reshape(snake_case_   , self.num_image_text_embeds   , -1    )

        # text
        A_    :			Optional[int]									=    self.text_proj(snake_case_    )

        return torch.cat([image_text_embeds, text_embeds]   , dim=1    )


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_ = 7_6_8   , snake_case_ = 7_6_8   , snake_case_ = 3_2   , ):

        """simple docstring"""

        super().__init__()

        A_    :			Any									=    num_image_text_embeds
        A_    :			Dict									=    nn.Linear(snake_case_   , self.num_image_text_embeds * cross_attention_dim    )
        A_    :			Union[str, Any]									=    nn.LayerNorm(snake_case_    )

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_    :			List[str]									=    image_embeds.shape[0]

        # image
        A_    :			str									=    self.image_embeds(snake_case_    )
        A_    :			List[str]									=    image_embeds.reshape(snake_case_   , self.num_image_text_embeds   , -1    )
        A_    :			Dict									=    self.norm(snake_case_    )
        return image_embeds


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_   , snake_case_=0.1    ):

        """simple docstring"""

        super().__init__()

        A_    :			Union[str, Any]									=    Timesteps(num_channels=2_5_6   , flip_sin_to_cos=snake_case_   , downscale_freq_shift=1    )
        A_    :			int									=    TimestepEmbedding(in_channels=2_5_6   , time_embed_dim=snake_case_    )
        A_    :			Union[str, Any]									=    LabelEmbedding(snake_case_   , snake_case_   , snake_case_    )

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_   , snake_case_=None    ):

        """simple docstring"""

        A_    :			str									=    self.time_proj(snake_case_    )
        A_    :			str									=    self.timestep_embedder(timesteps_proj.to(dtype=snake_case_    )    )  # (N, D)

        A_    :			List[Any]									=    self.class_embedder(snake_case_    )  # (N, D)

        A_    :			List[str]									=    timesteps_emb + class_labels  # (N, D)

        return conditioning


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_   , snake_case_ = 6_4    ):

        """simple docstring"""

        super().__init__()
        A_    :			str									=    nn.LayerNorm(snake_case_    )
        A_    :			Tuple									=    AttentionPooling(snake_case_   , snake_case_    )
        A_    :			Dict									=    nn.Linear(snake_case_   , snake_case_    )
        A_    :			Optional[int]									=    nn.LayerNorm(snake_case_    )

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_    :			int									=    self.norma(snake_case_    )
        A_    :			List[Any]									=    self.pool(snake_case_    )
        A_    :			List[Any]									=    self.proj(snake_case_    )
        A_    :			List[Any]									=    self.norma(snake_case_    )
        return hidden_states


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_ = 7_6_8   , snake_case_ = 7_6_8   , snake_case_ = 1_5_3_6    ):

        """simple docstring"""

        super().__init__()
        A_    :			Optional[Any]									=    nn.Linear(snake_case_   , snake_case_    )
        A_    :			Union[str, Any]									=    nn.LayerNorm(snake_case_    )
        A_    :			List[str]									=    nn.Linear(snake_case_   , snake_case_    )

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_    ):

        """simple docstring"""

        A_    :			Any									=    self.text_proj(snake_case_    )
        A_    :			Any									=    self.text_norm(snake_case_    )

        # image
        A_    :			Tuple									=    self.image_proj(snake_case_    )

        return time_image_embeds + time_text_embeds


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_ = 7_6_8   , snake_case_ = 1_5_3_6    ):

        """simple docstring"""

        super().__init__()
        A_    :			List[Any]									=    nn.Linear(snake_case_   , snake_case_    )
        A_    :			Union[str, Any]									=    nn.LayerNorm(snake_case_    )

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_    :			Optional[Any]									=    self.image_proj(snake_case_    )
        A_    :			Any									=    self.image_norm(snake_case_    )
        return time_image_embeds


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_ = 7_6_8   , snake_case_ = 1_5_3_6    ):

        """simple docstring"""

        super().__init__()
        A_    :			Optional[Any]									=    nn.Linear(snake_case_   , snake_case_    )
        A_    :			Optional[Any]									=    nn.LayerNorm(snake_case_    )
        A_    :			Any									=    nn.Sequential(
            nn.Convad(3   , 1_6   , 3   , padding=1    )   , nn.SiLU()   , nn.Convad(1_6   , 1_6   , 3   , padding=1    )   , nn.SiLU()   , nn.Convad(1_6   , 3_2   , 3   , padding=1   , stride=2    )   , nn.SiLU()   , nn.Convad(3_2   , 3_2   , 3   , padding=1    )   , nn.SiLU()   , nn.Convad(3_2   , 9_6   , 3   , padding=1   , stride=2    )   , nn.SiLU()   , nn.Convad(9_6   , 9_6   , 3   , padding=1    )   , nn.SiLU()   , nn.Convad(9_6   , 2_5_6   , 3   , padding=1   , stride=2    )   , nn.SiLU()   , nn.Convad(2_5_6   , 4   , 3   , padding=1    )   , )

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_    ):

        """simple docstring"""

        A_    :			int									=    self.image_proj(snake_case_    )
        A_    :			int									=    self.image_norm(snake_case_    )
        A_    :			int									=    self.input_hint_block(snake_case_    )
        return time_image_embeds, hint


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_   , snake_case_=None    ):

        """simple docstring"""

        super().__init__()
        A_    :			Dict									=    dtype
        A_    :			List[str]									=    nn.Parameter(torch.randn(1   , snake_case_    ) / embed_dim**0.5    )
        A_    :			List[str]									=    nn.Linear(snake_case_   , snake_case_   , dtype=self.dtype    )
        A_    :			Optional[int]									=    nn.Linear(snake_case_   , snake_case_   , dtype=self.dtype    )
        A_    :			str									=    nn.Linear(snake_case_   , snake_case_   , dtype=self.dtype    )
        A_    :			str									=    num_heads
        A_    :			int									=    embed_dim // self.num_heads

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_		,			A_		,			A_    :			Tuple									=    x.size()

        def shape(snake_case_    ):
          # (bs, length, width) --> (bs, length, n_heads, dim_per_head)
          A_    :			str									=    x.view(snake_case_   , -1   , self.num_heads   , self.dim_per_head    )
          # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
          A_    :			Optional[int]									=    x.transpose(1   , 2    )
          # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
          A_    :			List[Any]									=    x.reshape(bs * self.num_heads   , -1   , self.dim_per_head    )
          # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length)
          A_    :			List[Any]									=    x.transpose(1   , 2    )
          return x

        A_    :			int									=    x.mean(dim=1   , keepdim=snake_case_    ) + self.positional_embedding.to(x.dtype    )
        A_    :			Optional[Any]									=    torch.cat([class_token, x]   , dim=1    )  # (bs, length+1, width)

        # (bs*n_heads, class_token_length, dim_per_head)
        A_    :			Any									=    shape(self.q_proj(snake_case_    )    )
        # (bs*n_heads, length+class_token_length, dim_per_head)
        A_    :			Tuple									=    shape(self.k_proj(snake_case_    )    )
        A_    :			List[str]									=    shape(self.v_proj(snake_case_    )    )

        # (bs*n_heads, class_token_length, length+class_token_length):
        A_    :			str									=    1 / math.sqrt(math.sqrt(self.dim_per_head    )    )
        A_    :			str									=    torch.einsum('bct,bcs->bts'   , q * scale   , k * scale    )  # More stable with f16 than dividing afterwards
        A_    :			Tuple									=    torch.softmax(weight.float()   , dim=-1    ).type(weight.dtype    )

        # (bs*n_heads, dim_per_head, class_token_length)
        A_    :			int									=    torch.einsum('bts,bcs->bct'   , snake_case_   , snake_case_    )

        # (bs, length+1, width)
        A_    :			int									=    a.reshape(snake_case_   , -1   , 1    ).transpose(1   , 2    )

        return a[:, 0, :]  # cls_token



"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
    is_vision_available,
)


lowercase  		=			{
    """configuration_mobilevitv2""": [
        """MOBILEVITV2_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """MobileViTV2Config""",
        """MobileViTV2OnnxConfig""",
    ],
}


try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					lowercase  		=			[
					    """MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """MobileViTV2ForImageClassification""",
					    """MobileViTV2ForSemanticSegmentation""",
					    """MobileViTV2Model""",
					    """MobileViTV2PreTrainedModel""",
					]

if TYPE_CHECKING:
					from .configuration_mobilevitva import (
					    MOBILEVITV2_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    MobileViTVaConfig,
					    MobileViTVaOnnxConfig,
					)

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_mobilevitva import (
										    MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST,
										    MobileViTVaForImageClassification,
										    MobileViTVaForSemanticSegmentation,
										    MobileViTVaModel,
										    MobileViTVaPreTrainedModel,
										)

else:
					import sys

					lowercase  		=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





