






import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp







def 			lowercase__  (				A_:	List[str]     ,			A_:	str     ,			A_:	List[str]     ,			A_:	Union[str, Any]     ,			A_:	str = 4096		)  ->							Dict:







       """simple docstring"""






       __UpperCAmelCase    							=key.shape[-3:]
       __UpperCAmelCase    							=value.shape[-1]
       __UpperCAmelCase    							=min(A_     ,			A_		)
       __UpperCAmelCase    							=query / jnp.sqrt(A_		)

       @functools.partial(jax.checkpoint     ,			prevent_cse=A_		)
       def summarize_chunk(A_:	Tuple     ,			A_:	Optional[Any]     ,			A_:	Dict		):
              __UpperCAmelCase    							=jnp.einsum("""...qhd,...khd->...qhk"""     ,			A_     ,			A_     ,			precision=A_		)

              __UpperCAmelCase    							=jnp.max(A_     ,			axis=-1     ,			keepdims=A_		)
              __UpperCAmelCase    							=jax.lax.stop_gradient(A_		)
              __UpperCAmelCase    							=jnp.exp(attn_weights - max_score		)

              __UpperCAmelCase    							=jnp.einsum("""...vhf,...qhv->...qhf"""     ,			A_     ,			A_     ,			precision=A_		)
              __UpperCAmelCase    							=jnp.einsum("""...qhk->...qh"""     ,			A_		)

              return (exp_values, exp_weights.sum(axis=-1		), max_score)

       def chunk_scanner(A_:	List[Any]		):
              # julienne key array
              __UpperCAmelCase    							=jax.lax.dynamic_slice(
                  operand=A_     ,			start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0]     ,			slice_sizes=list(key.shape[:-3]		) + [key_chunk_size, num_heads, k_features]     ,			)

              # julienne value array
              __UpperCAmelCase    							=jax.lax.dynamic_slice(
                  operand=A_     ,			start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0]     ,			slice_sizes=list(value.shape[:-3]		) + [key_chunk_size, num_heads, v_features]     ,			)

              return summarize_chunk(A_     ,			A_     ,			A_		)

       __UpperCAmelCase    							=jax.lax.map(f=A_     ,			xs=jnp.arange(0     ,			A_     ,			A_		)		)

       __UpperCAmelCase    							=jnp.max(A_     ,			axis=0     ,			keepdims=A_		)
       __UpperCAmelCase    							=jnp.exp(chunk_max - global_max		)

       chunk_values *= jnp.expand_dims(A_     ,			axis=-1		)
       chunk_weights *= max_diffs

       __UpperCAmelCase    							=chunk_values.sum(axis=0		)
       __UpperCAmelCase    							=jnp.expand_dims(A_     ,			-1		).sum(axis=0		)

       return all_values / all_weights







def 			lowercase__  (				A_:	str     ,			A_:	Union[str, Any]     ,			A_:	Tuple     ,			A_:	str=jax.lax.Precision.HIGHEST     ,			A_:	Dict = 1024     ,			A_:	str = 4096		)  ->							int:







       """simple docstring"""






       __UpperCAmelCase    							=query.shape[-3:]

       def chunk_scanner(A_:	Tuple     ,			A_:	int		):
              # julienne query array
              __UpperCAmelCase    							=jax.lax.dynamic_slice(
                  operand=A_     ,			start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0]     ,			slice_sizes=list(query.shape[:-3]		) + [min(A_     ,			A_		), num_heads, q_features]     ,			)

              return (
                  chunk_idx + query_chunk_size,  # unused ignore it
                  _query_chunk_attention(
                      query=A_     ,			key=A_     ,			value=A_     ,			precision=A_     ,			key_chunk_size=A_		),
              )

       __UpperCAmelCase    							=jax.lax.scan(
           f=A_     ,			init=0     ,			xs=A_     ,			length=math.ceil(num_q / query_chunk_size		)		)

       return jnp.concatenate(A_     ,			axis=-3		)  # fuse the chunked result back



class 							_A (    nn.Module							):
    """simple docstring"""

    lowerCamelCase  :       Optional[int]   =   42
    lowerCamelCase  :       int   =   8
    lowerCamelCase  :       str   =   64
    lowerCamelCase  :       Optional[Any]   =   0.0
    lowerCamelCase  :       Tuple   =   False
    lowerCamelCase  :       Tuple   =   jnp.floataa



    def _a       (							self			:    Union[str, Any]			)			->       List[str]:
           __UpperCAmelCase    							=self.dim_head * self.heads
           __UpperCAmelCase    							=self.dim_head**-0.5

           # Weights were exported with old names {to_q, to_k, to_v, to_out}
           __UpperCAmelCase    							=nn.Dense(__A   ,    use_bias=__A   ,    dtype=self.dtype   ,    name="""to_q"""			)
           __UpperCAmelCase    							=nn.Dense(__A   ,    use_bias=__A   ,    dtype=self.dtype   ,    name="""to_k"""			)
           __UpperCAmelCase    							=nn.Dense(__A   ,    use_bias=__A   ,    dtype=self.dtype   ,    name="""to_v"""			)

           __UpperCAmelCase    							=nn.Dense(self.query_dim   ,    dtype=self.dtype   ,    name="""to_out_0"""			)
           __UpperCAmelCase    							=nn.Dropout(rate=self.dropout			)



    def _a       (							self			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    Tuple			)			->       Optional[Any]:
           __UpperCAmelCase    							=tensor.shape
           __UpperCAmelCase    							=self.heads
           __UpperCAmelCase    							=tensor.reshape(__A   ,    __A   ,    __A   ,    dim // head_size			)
           __UpperCAmelCase    							=jnp.transpose(__A   ,    (0, 2, 1, 3)			)
           __UpperCAmelCase    							=tensor.reshape(batch_size * head_size   ,    __A   ,    dim // head_size			)
           return tensor



    def _a       (							self			:    Any   ,    __SCREAMING_SNAKE_CASE			:    Dict			)			->       str:
           __UpperCAmelCase    							=tensor.shape
           __UpperCAmelCase    							=self.heads
           __UpperCAmelCase    							=tensor.reshape(batch_size // head_size   ,    __A   ,    __A   ,    __A			)
           __UpperCAmelCase    							=jnp.transpose(__A   ,    (0, 2, 1, 3)			)
           __UpperCAmelCase    							=tensor.reshape(batch_size // head_size   ,    __A   ,    dim * head_size			)
           return tensor



    def __call__(							self			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    int=None   ,    __SCREAMING_SNAKE_CASE			:    List[Any]=True			)			->       List[str]:
           __UpperCAmelCase    							=hidden_states if context is None else context

           __UpperCAmelCase    							=self.query(__A			)
           __UpperCAmelCase    							=self.key(__A			)
           __UpperCAmelCase    							=self.value(__A			)

           __UpperCAmelCase    							=self.reshape_heads_to_batch_dim(__A			)
           __UpperCAmelCase    							=self.reshape_heads_to_batch_dim(__A			)
           __UpperCAmelCase    							=self.reshape_heads_to_batch_dim(__A			)

           if self.use_memory_efficient_attention:
                  __UpperCAmelCase    							=query_states.transpose(1   ,    0   ,    2			)
                  __UpperCAmelCase    							=key_states.transpose(1   ,    0   ,    2			)
                  __UpperCAmelCase    							=value_states.transpose(1   ,    0   ,    2			)

                  # this if statement create a chunk size for each layer of the unet
                  # the chunk size is equal to the query_length dimension of the deepest layer of the unet

                  __UpperCAmelCase    							=query_states.shape[-3]
                  if flatten_latent_dim % 64 == 0:
                         __UpperCAmelCase    							=int(flatten_latent_dim / 64			)
                  elif flatten_latent_dim % 16 == 0:
                         __UpperCAmelCase    							=int(flatten_latent_dim / 16			)
                  elif flatten_latent_dim % 4 == 0:
                         __UpperCAmelCase    							=int(flatten_latent_dim / 4			)
                  else:
                         __UpperCAmelCase    							=int(__A			)

                  __UpperCAmelCase    							=jax_memory_efficient_attention(
                      __A   ,    __A   ,    __A   ,    query_chunk_size=__A   ,    key_chunk_size=4096 * 4			)

                  __UpperCAmelCase    							=hidden_states.transpose(1   ,    0   ,    2			)
           else:
                  # compute attentions
                  __UpperCAmelCase    							=jnp.einsum("""b i d, b j d->b i j"""   ,    __A   ,    __A			)
                  __UpperCAmelCase    							=attention_scores * self.scale
                  __UpperCAmelCase    							=nn.softmax(__A   ,    axis=2			)

                  # attend to values
                  __UpperCAmelCase    							=jnp.einsum("""b i j, b j d -> b i d"""   ,    __A   ,    __A			)

           __UpperCAmelCase    							=self.reshape_batch_dim_to_heads(__A			)
           __UpperCAmelCase    							=self.proj_attn(__A			)
           return self.dropout_layer(__A   ,    deterministic=__A			)




class 							_A (    nn.Module							):
    """simple docstring"""

    lowerCamelCase  :       List[Any]   =   42
    lowerCamelCase  :       int   =   42
    lowerCamelCase  :       Optional[Any]   =   42
    lowerCamelCase  :       Optional[int]   =   0.0
    lowerCamelCase  :       Union[str, Any]   =   False
    lowerCamelCase  :       Union[str, Any]   =   jnp.floataa
    lowerCamelCase  :       Optional[int]   =   False



    def _a       (							self			:    str			)			->       Union[str, Any]:
           __UpperCAmelCase    							=FlaxAttention(
               self.dim   ,    self.n_heads   ,    self.d_head   ,    self.dropout   ,    self.use_memory_efficient_attention   ,    dtype=self.dtype			)
           # cross attention
           __UpperCAmelCase    							=FlaxAttention(
               self.dim   ,    self.n_heads   ,    self.d_head   ,    self.dropout   ,    self.use_memory_efficient_attention   ,    dtype=self.dtype			)
           __UpperCAmelCase    							=FlaxFeedForward(dim=self.dim   ,    dropout=self.dropout   ,    dtype=self.dtype			)
           __UpperCAmelCase    							=nn.LayerNorm(epsilon=1e-5   ,    dtype=self.dtype			)
           __UpperCAmelCase    							=nn.LayerNorm(epsilon=1e-5   ,    dtype=self.dtype			)
           __UpperCAmelCase    							=nn.LayerNorm(epsilon=1e-5   ,    dtype=self.dtype			)
           __UpperCAmelCase    							=nn.Dropout(rate=self.dropout			)



    def __call__(							self			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Any   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]=True			)			->       int:
           __UpperCAmelCase    							=hidden_states
           if self.only_cross_attention:
                  __UpperCAmelCase    							=self.attna(self.norma(__A			)   ,    __A   ,    deterministic=__A			)
           else:
                  __UpperCAmelCase    							=self.attna(self.norma(__A			)   ,    deterministic=__A			)
           __UpperCAmelCase    							=hidden_states + residual

           # cross attention
           __UpperCAmelCase    							=hidden_states
           __UpperCAmelCase    							=self.attna(self.norma(__A			)   ,    __A   ,    deterministic=__A			)
           __UpperCAmelCase    							=hidden_states + residual

           # feed forward
           __UpperCAmelCase    							=hidden_states
           __UpperCAmelCase    							=self.ff(self.norma(__A			)   ,    deterministic=__A			)
           __UpperCAmelCase    							=hidden_states + residual

           return self.dropout_layer(__A   ,    deterministic=__A			)




class 							_A (    nn.Module							):
    """simple docstring"""

    lowerCamelCase  :       Tuple   =   42
    lowerCamelCase  :       str   =   42
    lowerCamelCase  :       Any   =   42
    lowerCamelCase  :       List[str]   =   1
    lowerCamelCase  :       Optional[Any]   =   0.0
    lowerCamelCase  :       Dict   =   False
    lowerCamelCase  :       Any   =   False
    lowerCamelCase  :       Union[str, Any]   =   jnp.floataa
    lowerCamelCase  :       int   =   False



    def _a       (							self			:    str			)			->       str:
           __UpperCAmelCase    							=nn.GroupNorm(num_groups=32   ,    epsilon=1e-5			)

           __UpperCAmelCase    							=self.n_heads * self.d_head
           if self.use_linear_projection:
                  __UpperCAmelCase    							=nn.Dense(__A   ,    dtype=self.dtype			)
           else:
                  __UpperCAmelCase    							=nn.Conv(
                      __A   ,    kernel_size=(1, 1)   ,    strides=(1, 1)   ,    padding="""VALID"""   ,    dtype=self.dtype   ,    )

           __UpperCAmelCase    							=[
               FlaxBasicTransformerBlock(
                   __A   ,    self.n_heads   ,    self.d_head   ,    dropout=self.dropout   ,    only_cross_attention=self.only_cross_attention   ,    dtype=self.dtype   ,    use_memory_efficient_attention=self.use_memory_efficient_attention   ,    )
               for _ in range(self.depth			)
           ]

           if self.use_linear_projection:
                  __UpperCAmelCase    							=nn.Dense(__A   ,    dtype=self.dtype			)
           else:
                  __UpperCAmelCase    							=nn.Conv(
                      __A   ,    kernel_size=(1, 1)   ,    strides=(1, 1)   ,    padding="""VALID"""   ,    dtype=self.dtype   ,    )

           __UpperCAmelCase    							=nn.Dropout(rate=self.dropout			)



    def __call__(							self			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    int   ,    __SCREAMING_SNAKE_CASE			:    List[str]=True			)			->       Optional[Any]:
           __UpperCAmelCase    							=hidden_states.shape
           __UpperCAmelCase    							=hidden_states
           __UpperCAmelCase    							=self.norm(__A			)
           if self.use_linear_projection:
                  __UpperCAmelCase    							=hidden_states.reshape(__A   ,    height * width   ,    __A			)
                  __UpperCAmelCase    							=self.proj_in(__A			)
           else:
                  __UpperCAmelCase    							=self.proj_in(__A			)
                  __UpperCAmelCase    							=hidden_states.reshape(__A   ,    height * width   ,    __A			)

           for transformer_block in self.transformer_blocks:
                  __UpperCAmelCase    							=transformer_block(__A   ,    __A   ,    deterministic=__A			)

           if self.use_linear_projection:
                  __UpperCAmelCase    							=self.proj_out(__A			)
                  __UpperCAmelCase    							=hidden_states.reshape(__A   ,    __A   ,    __A   ,    __A			)
           else:
                  __UpperCAmelCase    							=hidden_states.reshape(__A   ,    __A   ,    __A   ,    __A			)
                  __UpperCAmelCase    							=self.proj_out(__A			)

           __UpperCAmelCase    							=hidden_states + residual
           return self.dropout_layer(__A   ,    deterministic=__A			)




class 							_A (    nn.Module							):
    """simple docstring"""

    lowerCamelCase  :       List[Any]   =   42
    lowerCamelCase  :       List[str]   =   0.0
    lowerCamelCase  :       str   =   jnp.floataa



    def _a       (							self			:    List[Any]			)			->       int:
           __UpperCAmelCase    							=FlaxGEGLU(self.dim   ,    self.dropout   ,    self.dtype			)
           __UpperCAmelCase    							=nn.Dense(self.dim   ,    dtype=self.dtype			)



    def __call__(							self			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    str=True			)			->       Tuple:
           __UpperCAmelCase    							=self.net_a(__A   ,    deterministic=__A			)
           __UpperCAmelCase    							=self.net_a(__A			)
           return hidden_states




class 							_A (    nn.Module							):
    """simple docstring"""

    lowerCamelCase  :       Union[str, Any]   =   42
    lowerCamelCase  :       List[Any]   =   0.0
    lowerCamelCase  :       Union[str, Any]   =   jnp.floataa



    def _a       (							self			:    Tuple			)			->       List[Any]:
           __UpperCAmelCase    							=self.dim * 4
           __UpperCAmelCase    							=nn.Dense(inner_dim * 2   ,    dtype=self.dtype			)
           __UpperCAmelCase    							=nn.Dropout(rate=self.dropout			)



    def __call__(							self			:    str   ,    __SCREAMING_SNAKE_CASE			:    Any   ,    __SCREAMING_SNAKE_CASE			:    str=True			)			->       Optional[int]:
           __UpperCAmelCase    							=self.proj(__A			)
           __UpperCAmelCase    							=jnp.split(__A   ,    2   ,    axis=2			)
           return self.dropout_layer(hidden_linear * nn.gelu(__A			)   ,    deterministic=__A			)












import unittest

from transformers import ErnieMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       ErnieMForInformationExtraction,
       ErnieMForMultipleChoice,
       ErnieMForQuestionAnswering,
       ErnieMForSequenceClassification,
       ErnieMForTokenClassification,
       ErnieMModel,
   )
   from transformers.models.ernie_m.modeling_ernie_m import ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST



class 							_A :
    """simple docstring"""

    def __init__(							self			:    Any   ,    __SCREAMING_SNAKE_CASE			:    str   ,    __SCREAMING_SNAKE_CASE			:    List[str]=13   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]=7   ,    __SCREAMING_SNAKE_CASE			:    List[Any]=True   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]=True   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]=True   ,    __SCREAMING_SNAKE_CASE			:    str=99   ,    __SCREAMING_SNAKE_CASE			:    int=32   ,    __SCREAMING_SNAKE_CASE			:    str=5   ,    __SCREAMING_SNAKE_CASE			:    int=4   ,    __SCREAMING_SNAKE_CASE			:    List[str]=37   ,    __SCREAMING_SNAKE_CASE			:    int="gelu"   ,    __SCREAMING_SNAKE_CASE			:    Tuple=0.1   ,    __SCREAMING_SNAKE_CASE			:    List[str]=0.1   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]=512   ,    __SCREAMING_SNAKE_CASE			:    int=16   ,    __SCREAMING_SNAKE_CASE			:    int=2   ,    __SCREAMING_SNAKE_CASE			:    Any=0.02   ,    __SCREAMING_SNAKE_CASE			:    Dict=3   ,    __SCREAMING_SNAKE_CASE			:    Dict=4   ,    __SCREAMING_SNAKE_CASE			:    str=None   ,    )			->       Optional[Any]:
           __UpperCAmelCase    							=parent
           __UpperCAmelCase    							=batch_size
           __UpperCAmelCase    							=seq_length
           __UpperCAmelCase    							=is_training
           __UpperCAmelCase    							=use_input_mask
           __UpperCAmelCase    							=use_labels
           __UpperCAmelCase    							=vocab_size
           __UpperCAmelCase    							=hidden_size
           __UpperCAmelCase    							=num_hidden_layers
           __UpperCAmelCase    							=num_attention_heads
           __UpperCAmelCase    							=intermediate_size
           __UpperCAmelCase    							=hidden_act
           __UpperCAmelCase    							=hidden_dropout_prob
           __UpperCAmelCase    							=attention_probs_dropout_prob
           __UpperCAmelCase    							=max_position_embeddings
           __UpperCAmelCase    							=type_vocab_size
           __UpperCAmelCase    							=type_sequence_label_size
           __UpperCAmelCase    							=initializer_range
           __UpperCAmelCase    							=num_labels
           __UpperCAmelCase    							=num_choices
           __UpperCAmelCase    							=scope



    def _a       (							self			:    str			)			->       Optional[Any]:
           __UpperCAmelCase    							=ids_tensor([self.batch_size, self.seq_length]   ,    self.vocab_size			)

           __UpperCAmelCase    							=None
           if self.use_input_mask:
                  __UpperCAmelCase    							=random_attention_mask([self.batch_size, self.seq_length]			)

           __UpperCAmelCase    							=None
           __UpperCAmelCase    							=None
           __UpperCAmelCase    							=None
           if self.use_labels:
                  __UpperCAmelCase    							=ids_tensor([self.batch_size]   ,    self.type_sequence_label_size			)
                  __UpperCAmelCase    							=ids_tensor([self.batch_size, self.seq_length]   ,    self.num_labels			)
                  __UpperCAmelCase    							=ids_tensor([self.batch_size]   ,    self.num_choices			)

           __UpperCAmelCase    							=self.get_config()

           return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels



    def _a       (							self			:    Optional[Any]			)			->       Any:
           __UpperCAmelCase    							=ids_tensor([self.batch_size, self.seq_length]   ,    self.vocab_size			)

           __UpperCAmelCase    							=None
           if self.use_input_mask:
                  __UpperCAmelCase    							=random_attention_mask([self.batch_size, self.seq_length]			)
           __UpperCAmelCase    							=self.get_config()

           return config, input_ids, input_mask



    def _a       (							self			:    List[str]			)			->       List[str]:
           return ErnieMConfig(
               vocab_size=self.vocab_size   ,    hidden_size=self.hidden_size   ,    num_hidden_layers=self.num_hidden_layers   ,    num_attention_heads=self.num_attention_heads   ,    intermediate_size=self.intermediate_size   ,    hidden_act=self.hidden_act   ,    hidden_dropout_prob=self.hidden_dropout_prob   ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob   ,    max_position_embeddings=self.max_position_embeddings   ,    type_vocab_size=self.type_vocab_size   ,    initializer_range=self.initializer_range   ,    )



    def _a       (							self			:    str   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    str   ,    __SCREAMING_SNAKE_CASE			:    Dict			)			->       Optional[int]:
           __UpperCAmelCase    							=ErnieMModel(config=__SCREAMING_SNAKE_CASE			)
           model.to(__SCREAMING_SNAKE_CASE			)
           model.eval()
           __UpperCAmelCase    							=model(__SCREAMING_SNAKE_CASE   ,    return_dict=__SCREAMING_SNAKE_CASE			)
           self.parent.assertEqual(result.last_hidden_state.shape   ,    (self.batch_size, self.seq_length, self.hidden_size)			)



    def _a       (							self			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    Any   ,    __SCREAMING_SNAKE_CASE			:    Dict			)			->       int:
           __UpperCAmelCase    							=ErnieMForQuestionAnswering(config=__SCREAMING_SNAKE_CASE			)
           model.to(__SCREAMING_SNAKE_CASE			)
           model.eval()
           __UpperCAmelCase    							=model(
               __SCREAMING_SNAKE_CASE   ,    attention_mask=__SCREAMING_SNAKE_CASE   ,    start_positions=__SCREAMING_SNAKE_CASE   ,    end_positions=__SCREAMING_SNAKE_CASE   ,    )
           self.parent.assertEqual(result.start_logits.shape   ,    (self.batch_size, self.seq_length)			)
           self.parent.assertEqual(result.end_logits.shape   ,    (self.batch_size, self.seq_length)			)



    def _a       (							self			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    List[str]   ,    __SCREAMING_SNAKE_CASE			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]			)			->       Any:
           __UpperCAmelCase    							=ErnieMForInformationExtraction(config=__SCREAMING_SNAKE_CASE			)
           model.to(__SCREAMING_SNAKE_CASE			)
           model.eval()
           __UpperCAmelCase    							=torch.ones_like(__SCREAMING_SNAKE_CASE   ,    dtype=torch.floataa			)
           __UpperCAmelCase    							=model(
               __SCREAMING_SNAKE_CASE   ,    attention_mask=__SCREAMING_SNAKE_CASE   ,    start_positions=__SCREAMING_SNAKE_CASE   ,    end_positions=__SCREAMING_SNAKE_CASE   ,    )
           self.parent.assertEqual(result.start_logits.shape   ,    (self.batch_size, self.seq_length)			)
           self.parent.assertEqual(result.end_logits.shape   ,    (self.batch_size, self.seq_length)			)



    def _a       (							self			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    Any   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]			)			->       str:
           __UpperCAmelCase    							=self.num_labels
           __UpperCAmelCase    							=ErnieMForSequenceClassification(__SCREAMING_SNAKE_CASE			)
           model.to(__SCREAMING_SNAKE_CASE			)
           model.eval()
           __UpperCAmelCase    							=model(__SCREAMING_SNAKE_CASE   ,    attention_mask=__SCREAMING_SNAKE_CASE   ,    labels=__SCREAMING_SNAKE_CASE			)
           self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.num_labels)			)



    def _a       (							self			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    Dict			)			->       Dict:
           __UpperCAmelCase    							=self.num_labels
           __UpperCAmelCase    							=ErnieMForTokenClassification(config=__SCREAMING_SNAKE_CASE			)
           model.to(__SCREAMING_SNAKE_CASE			)
           model.eval()
           input_ids.to(__SCREAMING_SNAKE_CASE			)
           input_mask.to(__SCREAMING_SNAKE_CASE			)
           token_labels.to(__SCREAMING_SNAKE_CASE			)

           __UpperCAmelCase    							=model(__SCREAMING_SNAKE_CASE   ,    attention_mask=__SCREAMING_SNAKE_CASE   ,    labels=__SCREAMING_SNAKE_CASE			)

           self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.seq_length, self.num_labels)			)



    def _a       (							self			:    Optional[Any]   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    Optional[int]   ,    __SCREAMING_SNAKE_CASE			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    int   ,    __SCREAMING_SNAKE_CASE			:    Tuple   ,    __SCREAMING_SNAKE_CASE			:    Optional[Any]			)			->       List[Any]:
           __UpperCAmelCase    							=self.num_choices
           __UpperCAmelCase    							=ErnieMForMultipleChoice(config=__SCREAMING_SNAKE_CASE			)
           model.to(__SCREAMING_SNAKE_CASE			)
           model.eval()
           __UpperCAmelCase    							=input_ids.unsqueeze(1			).expand(-1   ,    self.num_choices   ,    -1			).contiguous()
           __UpperCAmelCase    							=input_mask.unsqueeze(1			).expand(-1   ,    self.num_choices   ,    -1			).contiguous()
           __UpperCAmelCase    							=model(
               __SCREAMING_SNAKE_CASE   ,    attention_mask=__SCREAMING_SNAKE_CASE   ,    labels=__SCREAMING_SNAKE_CASE   ,    )
           self.parent.assertEqual(result.logits.shape   ,    (self.batch_size, self.num_choices)			)



    def _a       (							self			:    Any			)			->       List[str]:
           __UpperCAmelCase    							=self.prepare_config_and_inputs()
           (
               (
               __UpperCAmelCase
           )							,   (
               __UpperCAmelCase
           )							,   (
               __UpperCAmelCase
           )							,   (
               __UpperCAmelCase
           )							,   (
               __UpperCAmelCase
           )							,   (
               __UpperCAmelCase
           )							,   
           )    							=config_and_inputs
           __UpperCAmelCase    							={"""input_ids""": input_ids, """attention_mask""": input_mask}
           return config, inputs_dict




@require_torch
class 							_A (    UpperCamelCase				,				UpperCamelCase				,				unittest.TestCase							):
    """simple docstring"""

    lowerCamelCase  :       Union[str, Any]   =   (
        (
            ErnieMModel,
            ErnieMForMultipleChoice,
            ErnieMForQuestionAnswering,
            ErnieMForSequenceClassification,
            ErnieMForTokenClassification,
        )
        if is_torch_available()
        else ()
    )
    lowerCamelCase  :       Optional[Any]   =   ()
    lowerCamelCase  :       str   =   (
        {
            'feature-extraction': ErnieMModel,
            'question-answering': ErnieMForQuestionAnswering,
            'text-classification': ErnieMForSequenceClassification,
            'token-classification': ErnieMForTokenClassification,
            'zero-shot': ErnieMForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    lowerCamelCase  :       Optional[int]   =   False



    def _a       (							self			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Dict   ,    __SCREAMING_SNAKE_CASE			:    Union[str, Any]   ,    __SCREAMING_SNAKE_CASE			:    List[Any]   ,    __SCREAMING_SNAKE_CASE			:    int   ,    __SCREAMING_SNAKE_CASE			:    int			)			->       Optional[int]:
           if pipeline_test_casse_name == "QAPipelineTests":
                  return True

           return False



    def _a       (							self			:    Optional[int]			)			->       List[Any]:
           __UpperCAmelCase    							=ErnieMModelTester(self			)
           __UpperCAmelCase    							=ConfigTester(self   ,    config_class=__SCREAMING_SNAKE_CASE   ,    hidden_size=37			)



    def _a       (							self			:    int			)			->       List[str]:
           self.config_tester.run_common_tests()



    def _a       (							self			:    str			)			->       str:
           __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE			)



    def _a       (							self			:    Optional[int]			)			->       List[str]:
           __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
           for type in ["absolute", "relative_key", "relative_key_query"]:
                  __UpperCAmelCase    							=type
                  self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE			)



    def _a       (							self			:    Optional[int]			)			->       str:
           __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_multiple_choice(*__SCREAMING_SNAKE_CASE			)



    def _a       (							self			:    List[str]			)			->       Union[str, Any]:
           __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_question_answering(*__SCREAMING_SNAKE_CASE			)



    def _a       (							self			:    Any			)			->       str:
           __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_sequence_classification(*__SCREAMING_SNAKE_CASE			)



    def _a       (							self			:    str			)			->       List[Any]:
           __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_information_extraction(*__SCREAMING_SNAKE_CASE			)



    def _a       (							self			:    List[Any]			)			->       List[str]:
           __UpperCAmelCase    							=self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_token_classification(*__SCREAMING_SNAKE_CASE			)



    @slow
    def _a       (							self			:    List[str]			)			->       Optional[Any]:
           for model_name in ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  __UpperCAmelCase    							=ErnieMModel.from_pretrained(__SCREAMING_SNAKE_CASE			)
                  self.assertIsNotNone(__SCREAMING_SNAKE_CASE			)




@require_torch
class 							_A (    unittest.TestCase							):
    """simple docstring"""

    @slow
    def _a       (							self			:    Tuple			)			->       Optional[Any]:
           __UpperCAmelCase    							=ErnieMModel.from_pretrained("""susnato/ernie-m-base_pytorch"""			)
           model.eval()
           __UpperCAmelCase    							=torch.tensor([[0, 1, 2, 3, 4, 5]]			)
           __UpperCAmelCase    							=model(__SCREAMING_SNAKE_CASE			)[0]

           # TODO Replace vocab size
           __UpperCAmelCase    							=768

           __UpperCAmelCase    							=torch.Size((1, 6, hidden_size)			)
           self.assertEqual(output.shape   ,    __SCREAMING_SNAKE_CASE			)

           __UpperCAmelCase    							=torch.tensor(
               [[[-0.0_012, 0.1_245, -0.0_214], [-0.0_742, 0.0_244, -0.0_771], [-0.0_333, 0.1_164, -0.1_554]]]			)

           self.assertTrue(torch.allclose(output[:, :3, :3]   ,    __SCREAMING_SNAKE_CASE   ,    atol=1e-3			)			)




