
"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase							 =							logging.get_logger(__name__)

_lowercase							 =							{
    '''tau/splinter-base''': '''https://huggingface.co/tau/splinter-base/resolve/main/config.json''',
    '''tau/splinter-base-qass''': '''https://huggingface.co/tau/splinter-base-qass/resolve/main/config.json''',
    '''tau/splinter-large''': '''https://huggingface.co/tau/splinter-large/resolve/main/config.json''',
    '''tau/splinter-large-qass''': '''https://huggingface.co/tau/splinter-large-qass/resolve/main/config.json''',
    # See all Splinter models at https://huggingface.co/models?filter=splinter
}

class 				__a    (   __a					):



							'''simple docstring'''







							_lowerCamelCase				: Optional[Any]					      =      """splinter"""
							def __init__(    self     ,						_lowerCamelCase=30_522     ,						_lowerCamelCase=768     ,						_lowerCamelCase=12     ,						_lowerCamelCase=12     ,						_lowerCamelCase=3_072     ,						_lowerCamelCase="gelu"     ,						_lowerCamelCase=0.1     ,						_lowerCamelCase=0.1     ,						_lowerCamelCase=512     ,						_lowerCamelCase=2     ,						_lowerCamelCase=0.02     ,						_lowerCamelCase=1e-12     ,						_lowerCamelCase=True     ,						_lowerCamelCase=0     ,						_lowerCamelCase=104     ,						**_lowerCamelCase     ,						) ->       Optional[int]:




										'''simple docstring'''
										super().__init__(pad_token_id=_lowerCamelCase     ,						**_lowerCamelCase					)

										__lowercase						      =					vocab_size
										__lowercase						      =					max_position_embeddings
										__lowercase						      =					hidden_size
										__lowercase						      =					num_hidden_layers
										__lowercase						      =					num_attention_heads
										__lowercase						      =					intermediate_size
										__lowercase						      =					hidden_act
										__lowercase						      =					hidden_dropout_prob
										__lowercase						      =					attention_probs_dropout_prob
										__lowercase						      =					initializer_range
										__lowercase						      =					type_vocab_size
										__lowercase						      =					layer_norm_eps
										__lowercase						      =					use_cache
										__lowercase						      =					question_token_id







"""simple docstring"""






import functools
import math

import flax.linen as nn
import jax
import jax.numpy as jnp



def     lowerCAmelCase__ (		__magic_name__ ,      __magic_name__ ,      __magic_name__ ,      __magic_name__ ,      __magic_name__ = 4_0_9_6			)	->Dict:
			__lowercase      ,				__lowercase      ,				__lowercase						      =					key.shape[-3:]
			__lowercase						      =					value.shape[-1]
			__lowercase						      =					min(__magic_name__ ,      __magic_name__			)
			__lowercase						      =					query / jnp.sqrt(__magic_name__			)

			@functools.partial(jax.checkpoint ,      prevent_cse=__magic_name__			)
			def summarize_chunk(__magic_name__ ,      __magic_name__ ,      __magic_name__			):
						__lowercase						      =					jnp.einsum("...qhd,...khd->...qhk" ,      __magic_name__ ,      __magic_name__ ,      precision=__magic_name__			)

						__lowercase						      =					jnp.max(__magic_name__ ,      axis=-1 ,      keepdims=__magic_name__			)
						__lowercase						      =					jax.lax.stop_gradient(__magic_name__			)
						__lowercase						      =					jnp.exp(attn_weights - max_score			)

						__lowercase						      =					jnp.einsum("...vhf,...qhv->...qhf" ,      __magic_name__ ,      __magic_name__ ,      precision=__magic_name__			)
						__lowercase						      =					jnp.einsum("...qhk->...qh" ,      __magic_name__			)

						return (exp_values, exp_weights.sum(axis=-1			), max_score)

			def chunk_scanner(__magic_name__			):
						# julienne key array
						__lowercase						      =					jax.lax.dynamic_slice(
						    operand=__magic_name__ ,      start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0] ,      slice_sizes=list(key.shape[:-3]			) + [key_chunk_size, num_heads, k_features] ,      )

						# julienne value array
						__lowercase						      =					jax.lax.dynamic_slice(
						    operand=__magic_name__ ,      start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0] ,      slice_sizes=list(value.shape[:-3]			) + [key_chunk_size, num_heads, v_features] ,      )

						return summarize_chunk(__magic_name__ ,      __magic_name__ ,      __magic_name__			)

			__lowercase      ,				__lowercase      ,				__lowercase						      =					jax.lax.map(f=__magic_name__ ,      xs=jnp.arange(0 ,      __magic_name__ ,      __magic_name__			)			)

			__lowercase						      =					jnp.max(__magic_name__ ,      axis=0 ,      keepdims=__magic_name__			)
			__lowercase						      =					jnp.exp(chunk_max - global_max			)

			chunk_values *= jnp.expand_dims(__magic_name__ ,      axis=-1			)
			chunk_weights *= max_diffs

			__lowercase						      =					chunk_values.sum(axis=0			)
			__lowercase						      =					jnp.expand_dims(__magic_name__ ,      -1			).sum(axis=0			)

			return all_values / all_weights



def     lowerCAmelCase__ (		__magic_name__ ,      __magic_name__ ,      __magic_name__ ,      __magic_name__=jax.lax.Precision.HIGHEST ,      __magic_name__ = 1_0_2_4 ,      __magic_name__ = 4_0_9_6			)	->List[str]:
			__lowercase      ,				__lowercase      ,				__lowercase						      =					query.shape[-3:]

			def chunk_scanner(__magic_name__ ,      __magic_name__			):
						# julienne query array
						__lowercase						      =					jax.lax.dynamic_slice(
						    operand=__magic_name__ ,      start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0] ,      slice_sizes=list(query.shape[:-3]			) + [min(__magic_name__ ,      __magic_name__			), num_heads, q_features] ,      )

						return (
						    chunk_idx + query_chunk_size,  # unused ignore it
						    _query_chunk_attention(
						        query=__magic_name__ ,      key=__magic_name__ ,      value=__magic_name__ ,      precision=__magic_name__ ,      key_chunk_size=__magic_name__			),
						)

			__lowercase      ,				__lowercase						      =					jax.lax.scan(
			    f=__magic_name__ ,      init=0 ,      xs=__magic_name__ ,      length=math.ceil(num_q / query_chunk_size			)			)

			return jnp.concatenate(__magic_name__ ,      axis=-3			)  # fuse the chunked result back

class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: int					      =      8
							_lowerCamelCase				: int					      =      64
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							def      SCREAMING_SNAKE_CASE	(    self					) ->       Any:




										'''simple docstring'''
										__lowercase						      =					self.dim_head * self.heads
										__lowercase						      =					self.dim_head**-0.5

										# Weights were exported with old names {to_q, to_k, to_v, to_out}
										__lowercase						      =					nn.Dense(_lowerCamelCase     ,						use_bias=_lowerCamelCase     ,						dtype=self.dtype     ,						name="to_q"					)
										__lowercase						      =					nn.Dense(_lowerCamelCase     ,						use_bias=_lowerCamelCase     ,						dtype=self.dtype     ,						name="to_k"					)
										__lowercase						      =					nn.Dense(_lowerCamelCase     ,						use_bias=_lowerCamelCase     ,						dtype=self.dtype     ,						name="to_v"					)

										__lowercase						      =					nn.Dense(self.query_dim     ,						dtype=self.dtype     ,						name="to_out_0"					)
										__lowercase						      =					nn.Dropout(rate=self.dropout					)
							def      SCREAMING_SNAKE_CASE	(    self     ,						_lowerCamelCase					) ->       Optional[int]:




										'''simple docstring'''
										__lowercase      ,				__lowercase      ,				__lowercase						      =					tensor.shape
										__lowercase						      =					self.heads
										__lowercase						      =					tensor.reshape(_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						dim // head_size					)
										__lowercase						      =					jnp.transpose(_lowerCamelCase     ,						(0, 2, 1, 3)					)
										__lowercase						      =					tensor.reshape(batch_size * head_size     ,						_lowerCamelCase     ,						dim // head_size					)
										return tensor
							def      SCREAMING_SNAKE_CASE	(    self     ,						_lowerCamelCase					) ->       str:




										'''simple docstring'''
										__lowercase      ,				__lowercase      ,				__lowercase						      =					tensor.shape
										__lowercase						      =					self.heads
										__lowercase						      =					tensor.reshape(batch_size // head_size     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase					)
										__lowercase						      =					jnp.transpose(_lowerCamelCase     ,						(0, 2, 1, 3)					)
										__lowercase						      =					tensor.reshape(batch_size // head_size     ,						_lowerCamelCase     ,						dim * head_size					)
										return tensor





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase=None     ,						_lowerCamelCase=True					) ->       List[str]:




										'''simple docstring'''
										__lowercase						      =					hidden_states if context is None else context

										__lowercase						      =					self.query(_lowerCamelCase					)
										__lowercase						      =					self.key(_lowerCamelCase					)
										__lowercase						      =					self.value(_lowerCamelCase					)

										__lowercase						      =					self.reshape_heads_to_batch_dim(_lowerCamelCase					)
										__lowercase						      =					self.reshape_heads_to_batch_dim(_lowerCamelCase					)
										__lowercase						      =					self.reshape_heads_to_batch_dim(_lowerCamelCase					)

										if self.use_memory_efficient_attention:
													__lowercase						      =					query_states.transpose(1     ,						0     ,						2					)
													__lowercase						      =					key_states.transpose(1     ,						0     ,						2					)
													__lowercase						      =					value_states.transpose(1     ,						0     ,						2					)

													# this if statement create a chunk size for each layer of the unet
													# the chunk size is equal to the query_length dimension of the deepest layer of the unet

													__lowercase						      =					query_states.shape[-3]
													if flatten_latent_dim % 64 == 0:
																__lowercase						      =					int(flatten_latent_dim / 64					)
													elif flatten_latent_dim % 16 == 0:
																__lowercase						      =					int(flatten_latent_dim / 16					)
													elif flatten_latent_dim % 4 == 0:
																__lowercase						      =					int(flatten_latent_dim / 4					)
													else:
																__lowercase						      =					int(_lowerCamelCase					)

													__lowercase						      =					jax_memory_efficient_attention(
													    _lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						query_chunk_size=_lowerCamelCase     ,						key_chunk_size=4_096 * 4					)

													__lowercase						      =					hidden_states.transpose(1     ,						0     ,						2					)
										else:
													# compute attentions
													__lowercase						      =					jnp.einsum("b i d, b j d->b i j"     ,						_lowerCamelCase     ,						_lowerCamelCase					)
													__lowercase						      =					attention_scores * self.scale
													__lowercase						      =					nn.softmax(_lowerCamelCase     ,						axis=2					)

													# attend to values
													__lowercase						      =					jnp.einsum("b i j, b j d -> b i d"     ,						_lowerCamelCase     ,						_lowerCamelCase					)

										__lowercase						      =					self.reshape_batch_dim_to_heads(_lowerCamelCase					)
										__lowercase						      =					self.proj_attn(_lowerCamelCase					)
										return self.dropout_layer(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							_lowerCamelCase				: bool					      =      False
							def      SCREAMING_SNAKE_CASE	(    self					) ->       str:




										'''simple docstring'''
										# self attention (or cross_attention if only_cross_attention is True)
										__lowercase						      =					FlaxAttention(
										    self.dim     ,						self.n_heads     ,						self.d_head     ,						self.dropout     ,						self.use_memory_efficient_attention     ,						dtype=self.dtype					)
										# cross attention
										__lowercase						      =					FlaxAttention(
										    self.dim     ,						self.n_heads     ,						self.d_head     ,						self.dropout     ,						self.use_memory_efficient_attention     ,						dtype=self.dtype					)
										__lowercase						      =					FlaxFeedForward(dim=self.dim     ,						dropout=self.dropout     ,						dtype=self.dtype					)
										__lowercase						      =					nn.LayerNorm(epsilon=1e-5     ,						dtype=self.dtype					)
										__lowercase						      =					nn.LayerNorm(epsilon=1e-5     ,						dtype=self.dtype					)
										__lowercase						      =					nn.LayerNorm(epsilon=1e-5     ,						dtype=self.dtype					)
										__lowercase						      =					nn.Dropout(rate=self.dropout					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       Optional[int]:




										'''simple docstring'''
										# self attention
										__lowercase						      =					hidden_states
										if self.only_cross_attention:
													__lowercase						      =					self.attna(self.norma(_lowerCamelCase					)     ,						_lowerCamelCase     ,						deterministic=_lowerCamelCase					)
										else:
													__lowercase						      =					self.attna(self.norma(_lowerCamelCase					)     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					hidden_states + residual

										# cross attention
										__lowercase						      =					hidden_states
										__lowercase						      =					self.attna(self.norma(_lowerCamelCase					)     ,						_lowerCamelCase     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					hidden_states + residual

										# feed forward
										__lowercase						      =					hidden_states
										__lowercase						      =					self.ff(self.norma(_lowerCamelCase					)     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					hidden_states + residual

										return self.dropout_layer(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: int
							_lowerCamelCase				: int					      =      1
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: bool					      =      False
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							_lowerCamelCase				: bool					      =      False
							def      SCREAMING_SNAKE_CASE	(    self					) ->       int:




										'''simple docstring'''
										__lowercase						      =					nn.GroupNorm(num_groups=32     ,						epsilon=1e-5					)

										__lowercase						      =					self.n_heads * self.d_head
										if self.use_linear_projection:
													__lowercase						      =					nn.Dense(_lowerCamelCase     ,						dtype=self.dtype					)
										else:
													__lowercase						      =					nn.Conv(
													    _lowerCamelCase     ,						kernel_size=(1, 1)     ,						strides=(1, 1)     ,						padding="VALID"     ,						dtype=self.dtype     ,						)

										__lowercase						      =					[
										    FlaxBasicTransformerBlock(
										        _lowerCamelCase     ,						self.n_heads     ,						self.d_head     ,						dropout=self.dropout     ,						only_cross_attention=self.only_cross_attention     ,						dtype=self.dtype     ,						use_memory_efficient_attention=self.use_memory_efficient_attention     ,						)
										    for _ in range(self.depth					)
										]

										if self.use_linear_projection:
													__lowercase						      =					nn.Dense(_lowerCamelCase     ,						dtype=self.dtype					)
										else:
													__lowercase						      =					nn.Conv(
													    _lowerCamelCase     ,						kernel_size=(1, 1)     ,						strides=(1, 1)     ,						padding="VALID"     ,						dtype=self.dtype     ,						)

										__lowercase						      =					nn.Dropout(rate=self.dropout					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       int:




										'''simple docstring'''
										__lowercase      ,				__lowercase      ,				__lowercase      ,				__lowercase						      =					hidden_states.shape
										__lowercase						      =					hidden_states
										__lowercase						      =					self.norm(_lowerCamelCase					)
										if self.use_linear_projection:
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						height * width     ,						_lowerCamelCase					)
													__lowercase						      =					self.proj_in(_lowerCamelCase					)
										else:
													__lowercase						      =					self.proj_in(_lowerCamelCase					)
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						height * width     ,						_lowerCamelCase					)

										for transformer_block in self.transformer_blocks:
													__lowercase						      =					transformer_block(_lowerCamelCase     ,						_lowerCamelCase     ,						deterministic=_lowerCamelCase					)

										if self.use_linear_projection:
													__lowercase						      =					self.proj_out(_lowerCamelCase					)
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase					)
										else:
													__lowercase						      =					hidden_states.reshape(_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase     ,						_lowerCamelCase					)
													__lowercase						      =					self.proj_out(_lowerCamelCase					)

										__lowercase						      =					hidden_states + residual
										return self.dropout_layer(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							def      SCREAMING_SNAKE_CASE	(    self					) ->       str:




										'''simple docstring'''
										# The second linear layer needs to be called
										# net_2 for now to match the index of the Sequential layer
										__lowercase						      =					FlaxGEGLU(self.dim     ,						self.dropout     ,						self.dtype					)
										__lowercase						      =					nn.Dense(self.dim     ,						dtype=self.dtype					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       int:




										'''simple docstring'''
										__lowercase						      =					self.net_a(_lowerCamelCase     ,						deterministic=_lowerCamelCase					)
										__lowercase						      =					self.net_a(_lowerCamelCase					)
										return hidden_states





class 				__a    (   nn.Module					):



							'''simple docstring'''







							_lowerCamelCase				: int
							_lowerCamelCase				: float					      =      0.0
							_lowerCamelCase				: jnp.dtype					      =      jnp.floataa
							def      SCREAMING_SNAKE_CASE	(    self					) ->       List[str]:




										'''simple docstring'''
										__lowercase						      =					self.dim * 4
										__lowercase						      =					nn.Dense(inner_dim * 2     ,						dtype=self.dtype					)
										__lowercase						      =					nn.Dropout(rate=self.dropout					)





							def __call__(    self     ,						_lowerCamelCase     ,						_lowerCamelCase=True					) ->       Union[str, Any]:




										'''simple docstring'''
										__lowercase						      =					self.proj(_lowerCamelCase					)
										__lowercase      ,				__lowercase						      =					jnp.split(_lowerCamelCase     ,						2     ,						axis=2					)
										return self.dropout_layer(hidden_linear * nn.gelu(_lowerCamelCase					)     ,						deterministic=_lowerCamelCase					)





