





"""simple docstring"""




import inspect
import unittest

from transformers import is_torch_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import MODEL_MAPPING, PoolFormerConfig, PoolFormerForImageClassification, PoolFormerModel
   from transformers.models.poolformer.modeling_poolformer import POOLFORMER_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
   from PIL import Image

   from transformers import PoolFormerImageProcessor

class 					_UpperCAmelCase    (  UpperCAmelCase__						):



      '''simple docstring'''

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Tuple									=    self.config_class(**self.inputs_dict    )
        self.parent.assertTrue(hasattr(snake_case_   , 'hidden_sizes'    )    )
        self.parent.assertTrue(hasattr(snake_case_   , 'num_encoder_blocks'    )    )


class 					_UpperCAmelCase    :



      '''simple docstring'''

      def __init__(	self   , snake_case_   , snake_case_=1_3   , snake_case_=6_4   , snake_case_=3   , snake_case_=4   , snake_case_=[2, 2, 2, 2]   , snake_case_=[8, 4, 2, 1]   , snake_case_=[1_6, 3_2, 6_4, 1_2_8]   , snake_case_=[1, 4, 8, 1_6]   , snake_case_=False   , snake_case_=True   , snake_case_="gelu"   , snake_case_=0.1   , snake_case_=0.02   , snake_case_=3   , snake_case_=None   , ):

        """simple docstring"""

        A_    :			Any									=    parent
        A_    :			Optional[Any]									=    batch_size
        A_    :			List[str]									=    image_size
        A_    :			Optional[int]									=    num_channels
        A_    :			Tuple									=    num_encoder_blocks
        A_    :			Tuple									=    sr_ratios
        A_    :			Optional[int]									=    depths
        A_    :			Dict									=    hidden_sizes
        A_    :			str									=    downsampling_rates
        A_    :			Dict									=    is_training
        A_    :			Union[str, Any]									=    use_labels
        A_    :			List[str]									=    hidden_act
        A_    :			Any									=    hidden_dropout_prob
        A_    :			int									=    initializer_range
        A_    :			List[str]									=    num_labels
        A_    :			str									=    scope

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			int									=    floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]    )

        A_    :			Optional[int]									=    None
        if self.use_labels:
          A_    :			Tuple									=    ids_tensor([self.batch_size, self.image_size, self.image_size]   , self.num_labels    )

        A_    :			Union[str, Any]									=    PoolFormerConfig(
            image_size=self.image_size   , num_channels=self.num_channels   , num_encoder_blocks=self.num_encoder_blocks   , depths=self.depths   , hidden_sizes=self.hidden_sizes   , hidden_act=self.hidden_act   , hidden_dropout_prob=self.hidden_dropout_prob   , initializer_range=self.initializer_range   , )

        return config, pixel_values, labels

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_   , snake_case_    ):

        """simple docstring"""

        A_    :			Any									=    PoolFormerModel(config=snake_case_    )
        model.to(snake_case_    )
        model.eval()
        A_    :			Dict									=    model(snake_case_    )
        A_    :			Union[str, Any]									=    self.image_size // 32.0
        self.parent.assertEqual(
            result.last_hidden_state.shape   , (self.batch_size, self.hidden_sizes[-1], expected_height, expected_width)    )

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Union[str, Any]									=    self.prepare_config_and_inputs()
        A_		,			A_		,			A_    :			Optional[int]									=    config_and_inputs
        A_    :			Any									=    {'pixel_values': pixel_values}
        return config, inputs_dict


@require_torch
class 					_UpperCAmelCase    (  UpperCAmelCase__     ,  UpperCAmelCase__     ,  unittest.TestCase						):



      '''simple docstring'''
      lowercase_ :					int             =  (PoolFormerModel, PoolFormerForImageClassification) if is_torch_available() else ()
      lowercase_ :					Any             =  (
          {"""feature-extraction""": PoolFormerModel, """image-classification""": PoolFormerForImageClassification}
          if is_torch_available()
          else {}
      )

      lowercase_ :					Union[str, Any]             =  False
      lowercase_ :					Optional[Any]             =  False
      lowercase_ :					List[Any]             =  False
      lowercase_ :					Any             =  False
      lowercase_ :					List[Any]             =  False

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Optional[int]									=    PoolFormerModelTester(self    )
        A_    :			Optional[Any]									=    PoolFormerConfigTester(self   , config_class=snake_case_    )

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        self.config_tester.run_common_tests()

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			List[str]									=    self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*snake_case_    )

      @unittest.skip('PoolFormer does not use inputs_embeds'    )
      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        pass

      @unittest.skip('PoolFormer does not have get_input_embeddings method and get_output_embeddings methods'    )
      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        pass

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        def check_hidden_states_output(snake_case_   , snake_case_   , snake_case_    ):
          A_    :			Dict									=    model_class(snake_case_    )
          model.to(snake_case_    )
          model.eval()

          with torch.no_grad():
            A_    :			str									=    model(**self._prepare_for_class(snake_case_   , snake_case_    )    )

          A_    :			List[Any]									=    outputs.hidden_states

          A_    :			str									=    self.model_tester.num_encoder_blocks
          self.assertEqual(len(snake_case_    )   , snake_case_    )

          # verify the first hidden states (first block)
          self.assertListEqual(
              list(hidden_states[0].shape[-3:]    )   , [
                  self.model_tester.hidden_sizes[0],
                  self.model_tester.image_size // 4,
                  self.model_tester.image_size // 4,
              ]   , )

        A_		,			A_    :			Tuple									=    self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          A_    :			List[str]									=    True
          check_hidden_states_output(snake_case_   , snake_case_   , snake_case_    )

          # check that output_hidden_states also work using config
          del inputs_dict["output_hidden_states"]
          A_    :			Optional[int]									=    True

          check_hidden_states_output(snake_case_   , snake_case_   , snake_case_    )

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        if not self.model_tester.is_training:
          return

        A_		,			A_    :			str									=    self.model_tester.prepare_config_and_inputs_for_common()
        A_    :			str									=    True

        for model_class in self.all_model_classes:
          if model_class in get_values(snake_case_    ):
            continue
          A_    :			List[str]									=    model_class(snake_case_    )
          model.to(snake_case_    )
          model.train()
          A_    :			Any									=    self._prepare_for_class(snake_case_   , snake_case_   , return_labels=snake_case_    )
          A_    :			List[str]									=    model(**snake_case_    ).loss
          loss.backward()

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_		,			A_    :			List[str]									=    self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
          A_    :			int									=    model_class(snake_case_    )
          A_    :			Any									=    inspect.signature(model.forward    )
          # signature.parameters is an OrderedDict => so arg_names order is deterministic
          A_    :			Optional[int]									=    [*signature.parameters.keys()]

          A_    :			List[Any]									=    ['pixel_values']
          self.assertListEqual(arg_names[:1]   , snake_case_    )

      @slow
      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        for model_name in POOLFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          A_    :			Any									=    PoolFormerModel.from_pretrained(snake_case_    )
          self.assertIsNotNone(snake_case_    )





def 		UpperCAmelCase__						(	):






  """simple docstring"""





  A_    :			List[str]									=    Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'   )
  return image





@require_torch
class 					_UpperCAmelCase    (  unittest.TestCase						):



      '''simple docstring'''

      @slow
      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			int									=    PoolFormerImageProcessor()
        A_    :			List[Any]									=    PoolFormerForImageClassification.from_pretrained('sail/poolformer_s12'    ).to(snake_case_    )

        A_    :			List[str]									=    image_processor(images=prepare_img()   , return_tensors='pt'    ).to(snake_case_    )

        # forward pass
        with torch.no_grad():
          A_    :			Any									=    model(**snake_case_    )

        # verify the logits
        A_    :			Union[str, Any]									=    torch.Size((1, 1_0_0_0)    )
        self.assertEqual(outputs.logits.shape   , snake_case_    )

        A_    :			List[Any]									=    torch.tensor([-0.61_13, 0.16_85, -0.04_92]    ).to(snake_case_    )
        self.assertTrue(torch.allclose(outputs.logits[0, :3]   , snake_case_   , atol=1E-4    )    )






"""simple docstring"""




import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp





def 		UpperCAmelCase__						(	_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase = 4096   ):






  """simple docstring"""





  A_		,			A_		,			A_    :			List[Any]									=    key.shape[-3:]
  A_    :			Dict									=    value.shape[-1]
  A_    :			Union[str, Any]									=    min(_UpperCAmelCase					,   _UpperCAmelCase   )
  A_    :			str									=    query / jnp.sqrt(_UpperCAmelCase   )

  @functools.partial(jax.checkpoint					,   prevent_cse=_UpperCAmelCase   )
  def summarize_chunk(_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase   ):
    A_    :			Any									=    jnp.einsum('...qhd,...khd->...qhk'					,   _UpperCAmelCase					,   _UpperCAmelCase					,   precision=_UpperCAmelCase   )

    A_    :			int									=    jnp.max(_UpperCAmelCase					,   axis=-1					,   keepdims=_UpperCAmelCase   )
    A_    :			List[str]									=    jax.lax.stop_gradient(_UpperCAmelCase   )
    A_    :			Dict									=    jnp.exp(attn_weights - max_score   )

    A_    :			int									=    jnp.einsum('...vhf,...qhv->...qhf'					,   _UpperCAmelCase					,   _UpperCAmelCase					,   precision=_UpperCAmelCase   )
    A_    :			Optional[Any]									=    jnp.einsum('...qhk->...qh'					,   _UpperCAmelCase   )

    return (exp_values, exp_weights.sum(axis=-1   ), max_score)

  def chunk_scanner(_UpperCAmelCase   ):
    # julienne key array
    A_    :			Dict									=    jax.lax.dynamic_slice(
        operand=_UpperCAmelCase					,   start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]					,   slice_sizes=list(key.shape[:-3]   ) + [key_chunk_size, num_heads, k_features]					,   )

    # julienne value array
    A_    :			Optional[Any]									=    jax.lax.dynamic_slice(
        operand=_UpperCAmelCase					,   start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]					,   slice_sizes=list(value.shape[:-3]   ) + [key_chunk_size, num_heads, v_features]					,   )

    return summarize_chunk(_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase   )

  A_		,			A_		,			A_    :			Dict									=    jax.lax.map(f=_UpperCAmelCase					,   xs=jnp.arange(0					,   _UpperCAmelCase					,   _UpperCAmelCase   )   )

  A_    :			int									=    jnp.max(_UpperCAmelCase					,   axis=0					,   keepdims=_UpperCAmelCase   )
  A_    :			Tuple									=    jnp.exp(chunk_max - global_max   )

  chunk_values *= jnp.expand_dims(_UpperCAmelCase					,   axis=-1   )
  chunk_weights *= max_diffs

  A_    :			List[Any]									=    chunk_values.sum(axis=0   )
  A_    :			Dict									=    jnp.expand_dims(_UpperCAmelCase					,   -1   ).sum(axis=0   )

  return all_values / all_weights





def 		UpperCAmelCase__						(	_UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase					,   _UpperCAmelCase=jax.lax.Precision.HIGHEST					,   _UpperCAmelCase = 1024					,   _UpperCAmelCase = 4096   ):






  """simple docstring"""





  A_		,			A_		,			A_    :			Any									=    query.shape[-3:]

  def chunk_scanner(_UpperCAmelCase					,   _UpperCAmelCase   ):
    # julienne query array
    A_    :			Tuple									=    jax.lax.dynamic_slice(
        operand=_UpperCAmelCase					,   start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]					,   slice_sizes=list(query.shape[:-3]   ) + [min(_UpperCAmelCase					,   _UpperCAmelCase   ), num_heads, q_features]					,   )

    return (
        chunk_idx + query_chunk_size,  # unused ignore it
        _query_chunk_attention(
            query=_UpperCAmelCase					,   key=_UpperCAmelCase					,   value=_UpperCAmelCase					,   precision=_UpperCAmelCase					,   key_chunk_size=_UpperCAmelCase   ),
    )

  A_		,			A_    :			Union[str, Any]									=    jax.lax.scan(
      f=_UpperCAmelCase					,   init=0					,   xs=_UpperCAmelCase					,   length=math.ceil(num_q / query_chunk_size   )   )

  return jnp.concatenate(_UpperCAmelCase					,   axis=-3   )  # fuse the chunked result back

class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					int             =  8
      lowercase_ :					int             =  64
      lowercase_ :					float             =  0.0
      lowercase_ :					bool             =  False
      lowercase_ :					jnp.dtype             =  jnp.floataa

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			List[str]									=    self.dim_head * self.heads
        A_    :			Tuple									=    self.dim_head**-0.5

        # Weights were exported with old names {to_q, to_k, to_v, to_out}
        A_    :			Any									=    nn.Dense(snake_case_   , use_bias=snake_case_   , dtype=self.dtype   , name='to_q'    )
        A_    :			int									=    nn.Dense(snake_case_   , use_bias=snake_case_   , dtype=self.dtype   , name='to_k'    )
        A_    :			Optional[int]									=    nn.Dense(snake_case_   , use_bias=snake_case_   , dtype=self.dtype   , name='to_v'    )

        A_    :			Union[str, Any]									=    nn.Dense(self.query_dim   , dtype=self.dtype   , name='to_out_0'    )
        A_    :			str									=    nn.Dropout(rate=self.dropout    )

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_		,			A_		,			A_    :			Union[str, Any]									=    tensor.shape
        A_    :			List[str]									=    self.heads
        A_    :			str									=    tensor.reshape(snake_case_   , snake_case_   , snake_case_   , dim // head_size    )
        A_    :			Optional[Any]									=    jnp.transpose(snake_case_   , (0, 2, 1, 3)    )
        A_    :			Any									=    tensor.reshape(batch_size * head_size   , snake_case_   , dim // head_size    )
        return tensor

      def    lowerCamelCase_     (	self   , snake_case_    ):

        """simple docstring"""

        A_		,			A_		,			A_    :			str									=    tensor.shape
        A_    :			Optional[Any]									=    self.heads
        A_    :			List[Any]									=    tensor.reshape(batch_size // head_size   , snake_case_   , snake_case_   , snake_case_    )
        A_    :			List[Any]									=    jnp.transpose(snake_case_   , (0, 2, 1, 3)    )
        A_    :			Any									=    tensor.reshape(batch_size // head_size   , snake_case_   , dim * head_size    )
        return tensor

      def __call__(	self   , snake_case_   , snake_case_=None   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Optional[Any]									=    hidden_states if context is None else context

        A_    :			str									=    self.query(snake_case_    )
        A_    :			Any									=    self.key(snake_case_    )
        A_    :			Union[str, Any]									=    self.value(snake_case_    )

        A_    :			int									=    self.reshape_heads_to_batch_dim(snake_case_    )
        A_    :			List[str]									=    self.reshape_heads_to_batch_dim(snake_case_    )
        A_    :			str									=    self.reshape_heads_to_batch_dim(snake_case_    )

        if self.use_memory_efficient_attention:
          A_    :			str									=    query_states.transpose(1   , 0   , 2    )
          A_    :			Optional[int]									=    key_states.transpose(1   , 0   , 2    )
          A_    :			Optional[Any]									=    value_states.transpose(1   , 0   , 2    )

          # this if statement create a chunk size for each layer of the unet
          # the chunk size is equal to the query_length dimension of the deepest layer of the unet

          A_    :			Union[str, Any]									=    query_states.shape[-3]
          if flatten_latent_dim % 6_4 == 0:
            A_    :			Any									=    int(flatten_latent_dim / 6_4    )
          elif flatten_latent_dim % 1_6 == 0:
            A_    :			Optional[int]									=    int(flatten_latent_dim / 1_6    )
          elif flatten_latent_dim % 4 == 0:
            A_    :			Dict									=    int(flatten_latent_dim / 4    )
          else:
            A_    :			Optional[int]									=    int(snake_case_    )

          A_    :			Any									=    jax_memory_efficient_attention(
              snake_case_   , snake_case_   , snake_case_   , query_chunk_size=snake_case_   , key_chunk_size=4_0_9_6 * 4    )

          A_    :			Any									=    hidden_states.transpose(1   , 0   , 2    )
        else:
          # compute attentions
          A_    :			str									=    jnp.einsum('b i d, b j d->b i j'   , snake_case_   , snake_case_    )
          A_    :			Dict									=    attention_scores * self.scale
          A_    :			List[str]									=    nn.softmax(snake_case_   , axis=2    )

          # attend to values
          A_    :			Optional[Any]									=    jnp.einsum('b i j, b j d -> b i d'   , snake_case_   , snake_case_    )

        A_    :			List[str]									=    self.reshape_batch_dim_to_heads(snake_case_    )
        A_    :			Tuple									=    self.proj_attn(snake_case_    )
        return self.dropout_layer(snake_case_   , deterministic=snake_case_    )


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					float             =  0.0
      lowercase_ :					bool             =  False
      lowercase_ :					jnp.dtype             =  jnp.floataa
      lowercase_ :					bool             =  False

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			List[str]									=    FlaxAttention(
            self.dim   , self.n_heads   , self.d_head   , self.dropout   , self.use_memory_efficient_attention   , dtype=self.dtype    )
        # cross attention
        A_    :			Optional[Any]									=    FlaxAttention(
            self.dim   , self.n_heads   , self.d_head   , self.dropout   , self.use_memory_efficient_attention   , dtype=self.dtype    )
        A_    :			Optional[Any]									=    FlaxFeedForward(dim=self.dim   , dropout=self.dropout   , dtype=self.dtype    )
        A_    :			Any									=    nn.LayerNorm(epsilon=1E-5   , dtype=self.dtype    )
        A_    :			Dict									=    nn.LayerNorm(epsilon=1E-5   , dtype=self.dtype    )
        A_    :			Optional[int]									=    nn.LayerNorm(epsilon=1E-5   , dtype=self.dtype    )
        A_    :			List[str]									=    nn.Dropout(rate=self.dropout    )

      def __call__(	self   , snake_case_   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Optional[int]									=    hidden_states
        if self.only_cross_attention:
          A_    :			Union[str, Any]									=    self.attna(self.norma(snake_case_    )   , snake_case_   , deterministic=snake_case_    )
        else:
          A_    :			Tuple									=    self.attna(self.norma(snake_case_    )   , deterministic=snake_case_    )
        A_    :			Optional[Any]									=    hidden_states + residual

        # cross attention
        A_    :			List[str]									=    hidden_states
        A_    :			List[str]									=    self.attna(self.norma(snake_case_    )   , snake_case_   , deterministic=snake_case_    )
        A_    :			Any									=    hidden_states + residual

        # feed forward
        A_    :			Dict									=    hidden_states
        A_    :			str									=    self.ff(self.norma(snake_case_    )   , deterministic=snake_case_    )
        A_    :			Optional[int]									=    hidden_states + residual

        return self.dropout_layer(snake_case_   , deterministic=snake_case_    )


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					int
      lowercase_ :					int             =  1
      lowercase_ :					float             =  0.0
      lowercase_ :					bool             =  False
      lowercase_ :					bool             =  False
      lowercase_ :					jnp.dtype             =  jnp.floataa
      lowercase_ :					bool             =  False

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Optional[Any]									=    nn.GroupNorm(num_groups=3_2   , epsilon=1E-5    )

        A_    :			Dict									=    self.n_heads * self.d_head
        if self.use_linear_projection:
          A_    :			Union[str, Any]									=    nn.Dense(snake_case_   , dtype=self.dtype    )
        else:
          A_    :			Any									=    nn.Conv(
              snake_case_   , kernel_size=(1, 1)   , strides=(1, 1)   , padding='VALID'   , dtype=self.dtype   , )

        A_    :			List[str]									=    [
            FlaxBasicTransformerBlock(
                snake_case_   , self.n_heads   , self.d_head   , dropout=self.dropout   , only_cross_attention=self.only_cross_attention   , dtype=self.dtype   , use_memory_efficient_attention=self.use_memory_efficient_attention   , )
            for _ in range(self.depth    )
        ]

        if self.use_linear_projection:
          A_    :			List[Any]									=    nn.Dense(snake_case_   , dtype=self.dtype    )
        else:
          A_    :			Dict									=    nn.Conv(
              snake_case_   , kernel_size=(1, 1)   , strides=(1, 1)   , padding='VALID'   , dtype=self.dtype   , )

        A_    :			List[str]									=    nn.Dropout(rate=self.dropout    )

      def __call__(	self   , snake_case_   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_		,			A_		,			A_		,			A_    :			Any									=    hidden_states.shape
        A_    :			Any									=    hidden_states
        A_    :			List[Any]									=    self.norm(snake_case_    )
        if self.use_linear_projection:
          A_    :			str									=    hidden_states.reshape(snake_case_   , height * width   , snake_case_    )
          A_    :			Optional[Any]									=    self.proj_in(snake_case_    )
        else:
          A_    :			List[str]									=    self.proj_in(snake_case_    )
          A_    :			List[Any]									=    hidden_states.reshape(snake_case_   , height * width   , snake_case_    )

        for transformer_block in self.transformer_blocks:
          A_    :			int									=    transformer_block(snake_case_   , snake_case_   , deterministic=snake_case_    )

        if self.use_linear_projection:
          A_    :			Union[str, Any]									=    self.proj_out(snake_case_    )
          A_    :			Any									=    hidden_states.reshape(snake_case_   , snake_case_   , snake_case_   , snake_case_    )
        else:
          A_    :			int									=    hidden_states.reshape(snake_case_   , snake_case_   , snake_case_   , snake_case_    )
          A_    :			Union[str, Any]									=    self.proj_out(snake_case_    )

        A_    :			Union[str, Any]									=    hidden_states + residual
        return self.dropout_layer(snake_case_   , deterministic=snake_case_    )


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					float             =  0.0
      lowercase_ :					jnp.dtype             =  jnp.floataa

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			List[str]									=    FlaxGEGLU(self.dim   , self.dropout   , self.dtype    )
        A_    :			int									=    nn.Dense(self.dim   , dtype=self.dtype    )

      def __call__(	self   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Dict									=    self.net_a(snake_case_   , deterministic=snake_case_    )
        A_    :			Any									=    self.net_a(snake_case_    )
        return hidden_states


class 					_UpperCAmelCase    (  nn.Module						):



      '''simple docstring'''
      lowercase_ :					int
      lowercase_ :					float             =  0.0
      lowercase_ :					jnp.dtype             =  jnp.floataa

      def    lowerCamelCase_     (	self    ):

        """simple docstring"""

        A_    :			Optional[int]									=    self.dim * 4
        A_    :			List[str]									=    nn.Dense(inner_dim * 2   , dtype=self.dtype    )
        A_    :			int									=    nn.Dropout(rate=self.dropout    )

      def __call__(	self   , snake_case_   , snake_case_=True    ):

        """simple docstring"""

        A_    :			Optional[Any]									=    self.proj(snake_case_    )
        A_		,			A_    :			Any									=    jnp.split(snake_case_   , 2   , axis=2    )
        return self.dropout_layer(hidden_linear * nn.gelu(snake_case_    )   , deterministic=snake_case_    )